diff --git a/acceptance/air/cancel/output.txt b/acceptance/air/cancel/output.txt index 58a33d92bea..22d4e79b830 100644 --- a/acceptance/air/cancel/output.txt +++ b/acceptance/air/cancel/output.txt @@ -1,9 +1,11 @@ === cancel by id (text) + >>> [CLI] air cancel 123 Successfully requested cancellation for run 123 === cancel by id (json) + >>> [CLI] air cancel 123 -o json { "v": 1, @@ -16,12 +18,14 @@ Successfully requested cancellation for run 123 } === cancel multiple ids + >>> [CLI] air cancel 123 456 Successfully requested cancellation for run 123 Successfully requested cancellation for run 456 Successfully requested cancellation for 2 run(s). === cancel --all + >>> [CLI] air cancel --all -y Searching active runs for [USERNAME] in [DATABRICKS_URL]... Successfully requested cancellation for run 334747067049496 diff --git a/acceptance/air/config-help/output.txt b/acceptance/air/config-help/output.txt index 253e8380004..75dad55277c 100644 --- a/acceptance/air/config-help/output.txt +++ b/acceptance/air/config-help/output.txt @@ -1,5 +1,6 @@ === command help + >>> [CLI] air run --help Submit a training workload to Databricks serverless GPU compute. @@ -32,6 +33,7 @@ Global Flags: -t, --target string bundle target to use (if applicable) === schema overview + >>> [CLI] air run -h config config The run YAML schema. Pass a field path for details, e.g. config.compute.accelerator_type. @@ -70,6 +72,7 @@ config Use "-h config." for details on a field. === nested object lists its fields + >>> [CLI] air run -h config.compute config.compute Which accelerators to run on and how many. @@ -88,6 +91,7 @@ config.compute Use "-h config.compute." for details on a field. === new submission fields are documented + >>> [CLI] air run -h config.mlflow_artifact_location config.mlflow_artifact_location DBFS location where MLflow artifacts are written. A /Volumes path is normalized to dbfs:/Volumes/... . @@ -124,6 +128,7 @@ config.environment.version Required: no === leaf field + >>> [CLI] air run -h config.compute.accelerator_type config.compute.accelerator_type Which accelerator to run on, e.g. GPU_1xA10. See https://docs.databricks.com/aws/en/machine-learning/ai-runtime/cli/yaml-config#reference for the current list of supported GPU types. Matched case-sensitively. @@ -137,6 +142,7 @@ config.compute.accelerator_type Required: no === code source fields + >>> [CLI] air run -h config.code_source config.code_source Local code to upload and make available to the run. @@ -150,6 +156,7 @@ config.code_source Use "-h config.code_source." for details on a field. === union field + >>> [CLI] air run -h config.code_source.git.remote config.code_source.git.remote No longer supported: the snapshot archives your local copy. Only false is accepted; use commit to pin a revision. @@ -158,6 +165,7 @@ config.code_source.git.remote Required: no === git-pinned subdirectory documents root_path scoping + >>> [CLI] air run -h config.code_source.root_path config.code_source.root_path Root of the code source to archive. A git-pinned subdirectory packages only that subtree. @@ -166,6 +174,7 @@ config.code_source.root_path Required: when code_source is set === the config. prefix is optional + >>> [CLI] air run -h compute.num_accelerators config.compute.num_accelerators Total number of GPUs to allocate. Must be a positive multiple of the accelerator type's per-node GPU count. See https://docs.databricks.com/aws/en/machine-learning/ai-runtime/cli/yaml-config#reference for supported GPU types. @@ -174,11 +183,13 @@ config.compute.num_accelerators Required: no === unknown field suggests the near match + >>> [CLI] air run -h config.compute.acclerator_type Error: unknown config field "config.compute.acclerator_type"; did you mean "accelerator_type"? fields under "config.compute" are: accelerator_type, num_accelerators, pool_id, priority_class === free-form map keys are not schema fields + >>> [CLI] air run -h config.parameters.learning_rate Error: "config.parameters" holds user-defined keys, so "learning_rate" is not part of the schema; see "config.parameters" instead diff --git a/acceptance/air/convert-to-dabs/output.txt b/acceptance/air/convert-to-dabs/output.txt index 4b80cc0e0c5..7a0beec224b 100644 --- a/acceptance/air/convert-to-dabs/output.txt +++ b/acceptance/air/convert-to-dabs/output.txt @@ -1,5 +1,6 @@ === convert an AIR run YAML into a DABs bundle (in place, next to the source) + >>> [CLI] air convert-to-dabs train.yaml Wrote a Databricks Asset Bundle to .: databricks.yml @@ -20,6 +21,7 @@ job and its uploaded files with: [CLI] bundle destroy === emitted databricks.yml (code_source packaged as a tgz artifact; no code is copied) + >>> cat databricks.yml bundle: name: torchrun-a10-smoke-test @@ -65,11 +67,13 @@ resources: environment_version: "6" === the generated command.sh carries the run command + >>> cat generated_artifacts/command.sh cd $CODE_SOURCE_PATH torchrun --nproc_per_node=1 train.py === the emitted bundle validates + >>> [CLI] bundle validate Name: torchrun-a10-smoke-test Target: dev @@ -80,12 +84,14 @@ Workspace: Validation OK! === re-converting refuses to clobber the generated bundle + >>> [CLI] air convert-to-dabs train.yaml Error: databricks.yml already exists in .; pass --force to overwrite or remove it Exit code: 1 === --force overwrites it + >>> [CLI] air convert-to-dabs train.yaml --force Wrote a Databricks Asset Bundle to .: databricks.yml @@ -106,6 +112,7 @@ job and its uploaded files with: [CLI] bundle destroy === convert a B300 run using the current SDK + >>> [CLI] air convert-to-dabs b300.yaml --output-dir generated-b300 Wrote a Databricks Asset Bundle to generated-b300: databricks.yml @@ -127,6 +134,7 @@ job and its uploaded files with: [CLI] bundle destroy === the B300 bundle preserves its accelerator and AI Runtime version + >>> cat generated-b300/databricks.yml bundle: name: b300-smoke-test @@ -159,6 +167,7 @@ resources: environment_version: "6" === the current SDK recognizes B300 so bundle validation is clean + >>> [CLI] bundle validate Name: b300-smoke-test Target: dev @@ -169,6 +178,7 @@ Workspace: Validation OK! === databricks_ai versions use a base environment + >>> [CLI] air convert-to-dabs databricks-ai.yaml --force Wrote a Databricks Asset Bundle to .: databricks.yml @@ -192,6 +202,7 @@ job and its uploaded files with: base_environment: workspace-base-environments/databricks_ai_v5 === the Databricks AI bundle validates + >>> [CLI] bundle validate Name: databricks-ai-v5-smoke-test Target: dev diff --git a/acceptance/air/get-ai-runtime/output.txt b/acceptance/air/get-ai-runtime/output.txt index 096b127a6f1..0485ab36852 100644 --- a/acceptance/air/get-ai-runtime/output.txt +++ b/acceptance/air/get-ai-runtime/output.txt @@ -1,5 +1,6 @@ === get (text) + >>> [CLI] air get 123 ╭─ Configuration ────────────────────────────────────────────────╮ @@ -35,6 +36,7 @@ Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID] MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 === get (json) + >>> [CLI] air get 123 -o json { "v": 1, diff --git a/acceptance/air/get/output.txt b/acceptance/air/get/output.txt index 2a2809cf255..df8398a83d0 100644 --- a/acceptance/air/get/output.txt +++ b/acceptance/air/get/output.txt @@ -1,5 +1,6 @@ === get (text) + >>> [CLI] air get 123 ╭─ Configuration ────────────────────────────────────────────────╮ @@ -35,6 +36,7 @@ Run URL: [DATABRICKS_URL]/jobs/runs/123?o=[WORKSPACE_ID] MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 === get (json) + >>> [CLI] air get 123 -o json { "v": 1, @@ -54,12 +56,14 @@ MLflow URL: [DATABRICKS_URL]/ml/experiments/exp1/runs/run1 } === invalid run id + >>> [CLI] air get notanumber Error: invalid JOB_RUN_ID "notanumber": must be a positive integer Exit code: 1 === invalid run id (json) + >>> [CLI] air get notanumber -o json { "v": 1, diff --git a/acceptance/air/help/output.txt b/acceptance/air/help/output.txt index 5bb6f40b4f7..e4cac86c73d 100644 --- a/acceptance/air/help/output.txt +++ b/acceptance/air/help/output.txt @@ -1,5 +1,6 @@ === help + >>> [CLI] air --help Run and manage AI Runtime training workloads on Databricks serverless GPU compute. @@ -30,6 +31,7 @@ Global Flags: Use "databricks air [command] --help" for more information about a command. === list help + >>> [CLI] air list --help List active runs for the current user. @@ -66,6 +68,7 @@ Global Flags: -t, --target string bundle target to use (if applicable) === get help + >>> [CLI] air get --help Show status, configuration, and timing details for a specified run. @@ -84,6 +87,7 @@ Global Flags: -t, --target string bundle target to use (if applicable) === pools help + >>> [CLI] air pools --help View provisioned GPU pools @@ -107,6 +111,7 @@ Global Flags: Use "databricks air pools [command] --help" for more information about a command. === images help + >>> [CLI] air images --help Manage container images for AI Runtime @@ -129,6 +134,7 @@ Global Flags: Use "databricks air images [command] --help" for more information about a command. === images push help + >>> [CLI] air images push --help This CLI command is in Preview. For current feature status and requirements, see: AWS: https://docs.databricks.com/aws/en/machine-learning/ai-runtime/cli/docker-images @@ -162,6 +168,7 @@ Global Flags: -t, --target string bundle target to use (if applicable) === logs help + >>> [CLI] air logs --help Stream logs from an active run until it completes, or fetch logs from a completed run. @@ -191,6 +198,7 @@ Global Flags: -t, --target string bundle target to use (if applicable) === cancel help + >>> [CLI] air cancel --help Cancel one or more runs by job run ID, or use --all to cancel all active runs submitted by the current user. diff --git a/acceptance/air/image-push/output.txt b/acceptance/air/image-push/output.txt index 03c49f36d40..7f2dce10ac9 100644 --- a/acceptance/air/image-push/output.txt +++ b/acceptance/air/image-push/output.txt @@ -1,5 +1,6 @@ === push image + >>> databricks air images push -p docker-test -s nvidia/cuda:13.4.1 --catalog my-catalog --schema my-schema --artifact cuda:13.4.1-cudnn-devel-ubuntu26.04 Warning: This feature is in Preview. APIs may change, and the workspace must enable the Preview features. Pushing nvidia/cuda:13.4.1 as my-catalog.my-schema.cuda:13.4.1-cudnn-devel-ubuntu26.04 using profile docker-test @@ -35,6 +36,7 @@ tag nvidia/cuda:13.4.1 [WORKSPACE_ID].container.us-west-2.localhost/my-catalog.m push [WORKSPACE_ID].container.us-west-2.localhost/my-catalog.my-schema.cuda:13.4.1-cudnn-devel-ubuntu26.04 === missing image is pulled for AIR; destination defaults to source + >>> databricks air images push -p docker-test -s example/image:v1 --catalog main --schema training Warning: This feature is in Preview. APIs may change, and the workspace must enable the Preview features. Pushing example/image:v1 as main.training.image:v1 using profile docker-test @@ -50,6 +52,7 @@ Pushed image. Now you can set `environment.unity_catalog_image = main.training.image:v1` in your YAML config and create a workload with `databricks air run -f config.yaml`. === forced pull and JSON output + >>> databricks air images push -p docker-test -s example/image:v1 --catalog main --schema training --pull -o json Warning: This feature is in Preview. APIs may change, and the workspace must enable the Preview features. Pushing example/image:v1 as main.training.image:v1 using profile docker-test @@ -73,6 +76,7 @@ Pushed image. } === daemon failure must not pull + >>> musterr databricks air images push -p docker-test -s example/image:v1 --catalog main --schema training Warning: This feature is in Preview. APIs may change, and the workspace must enable the Preview features. Pushing example/image:v1 as main.training.image:v1 using profile docker-test @@ -83,6 +87,7 @@ Cannot connect to the Docker daemon Error: check local source image example/image:v1 (ensure Docker is running and accessible): exit status 1 === inspect failure must not pull + >>> musterr databricks air images push -p docker-test -s example/image:v1 --catalog main --schema training Warning: This feature is in Preview. APIs may change, and the workspace must enable the Preview features. Pushing example/image:v1 as main.training.image:v1 using profile docker-test @@ -94,6 +99,7 @@ Docker inspect failed Error: inspect source image example/image:v1: exit status 1 === incompatible local image must not be pushed + >>> musterr databricks air images push -p docker-test -s example/image:v1 --catalog main --schema training Warning: This feature is in Preview. APIs may change, and the workspace must enable the Preview features. Pushing example/image:v1 as main.training.image:v1 using profile docker-test @@ -104,6 +110,7 @@ Using local source image example/image:v1 Error: source image example/image:v1 has platform "linux/arm64"; AIR requires linux/amd64: use --pull to fetch a compatible image, or rebuild the source for linux/amd64 === incompatible pulled image must not be pushed + >>> musterr databricks air images push -p docker-test -s example/image:v1 --catalog main --schema training --pull Warning: This feature is in Preview. APIs may change, and the workspace must enable the Preview features. Pushing example/image:v1 as main.training.image:v1 using profile docker-test @@ -115,6 +122,7 @@ Docker pull progress Error: source image example/image:v1 has platform "linux/arm64"; AIR requires linux/amd64: use --pull to fetch a compatible image, or rebuild the source for linux/amd64 === pull failure must not be pushed + >>> musterr databricks air images push -p docker-test -s example/image:v1 --catalog main --schema training --pull Warning: This feature is in Preview. APIs may change, and the workspace must enable the Preview features. Pushing example/image:v1 as main.training.image:v1 using profile docker-test @@ -126,6 +134,7 @@ Docker pull failed Error: pull source image example/image:v1: exit status 1 === invalid destination and removed flags + >>> musterr databricks air images push -p docker-test -s example/image:v1 --catalog main --schema training --artifact invalid.name Warning: This feature is in Preview. APIs may change, and the workspace must enable the Preview features. Error: invalid artifact name "invalid.name": use lowercase letters and digits separated by one or two underscores or one or more hyphens @@ -177,6 +186,7 @@ Global Flags: === reject oversized catalog.schema.artifact before Docker operations + >>> musterr databricks air images push -p docker-test -s example/image:v1 --catalog main --schema training --artifact aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa Warning: This feature is in Preview. APIs may change, and the workspace must enable the Preview features. Error: invalid image repository name: catalog.schema.artifact must be 255 characters or less diff --git a/acceptance/air/image-push/script b/acceptance/air/image-push/script index bd641b65d9e..4863b27b366 100644 --- a/acceptance/air/image-push/script +++ b/acceptance/air/image-push/script @@ -12,10 +12,10 @@ trace databricks air images push \ --schema my-schema \ --artifact cuda:13.4.1-cudnn-devel-ubuntu26.04 -title "inferred region and workspace ID requests\n" +title "inferred region and workspace ID requests" trace print_requests.py --get //api -title "docker commands\n" +title "docker commands" cat "$DOCKER_LOG" title "missing image is pulled for AIR; destination defaults to source" @@ -59,5 +59,5 @@ cp "$DOCKER_LOG" "$HOME/docker.before-length.log" trace musterr databricks air images push -p docker-test -s example/image:v1 --catalog main --schema training --artifact "$(cat oversized-artifact.txt)" cmp "$DOCKER_LOG" "$HOME/docker.before-length.log" -title "all docker commands\n" +title "all docker commands" cat "$DOCKER_LOG" diff --git a/acceptance/air/list/output.txt b/acceptance/air/list/output.txt index 9dfa0b22f24..02f4f625742 100644 --- a/acceptance/air/list/output.txt +++ b/acceptance/air/list/output.txt @@ -1,10 +1,12 @@ === list (text) + >>> [CLI] air list Run ID Experiment Status Started Duration Progress MLflow User Accelerators 334747067049496 qwen-train ● SUCCESS [TIMESTAMP] 12s - qwen-train-001 [USERNAME] 8x H100 === list (json) + >>> [CLI] air list -o json { "v": 1, @@ -24,11 +26,13 @@ } === list --all-status (text, via AiTrainingService index) + >>> [CLI] air list --all-status Run ID Experiment Status Started Duration Progress MLflow User Accelerators 334747067049496 qwen-train ● SUCCESS [TIMESTAMP] 12s - qwen-train-001 [USERNAME] 8x H100 === list --all-status (json) + >>> [CLI] air list --all-status -o json { "v": 1, diff --git a/acceptance/air/logs-download/output.txt b/acceptance/air/logs-download/output.txt index 4dedd822aa2..44bf1ec1e8b 100644 --- a/acceptance/air/logs-download/output.txt +++ b/acceptance/air/logs-download/output.txt @@ -1,15 +1,18 @@ === download-to reports no logs when none are available + >>> [CLI] air logs 123 --download-to dl-logs No logs available for run 123. Run terminated in state SUCCESS === download-to with an out-of-range node is rejected + >>> [CLI] air logs 123 --download-to dl-logs --node 5 Error: invalid --node 5: run has 2 node(s), indexed 0 to 1 Exit code: 1 === download-to cannot be combined with --tail + >>> [CLI] air logs 123 --download-to dl-logs --tail 50 Error: --download-to writes complete logs, so it cannot be combined with --tail or --minutes diff --git a/acceptance/air/logs-mlflow-fallback/output.txt b/acceptance/air/logs-mlflow-fallback/output.txt index e783e9a1ba7..75459d38945 100644 --- a/acceptance/air/logs-mlflow-fallback/output.txt +++ b/acceptance/air/logs-mlflow-fallback/output.txt @@ -1,8 +1,10 @@ === logs falls back to mlflow (no logs) + >>> [CLI] air logs 123 No logs available for run 123. Run terminated in state SUCCESS === logs falls back to mlflow (json) + >>> [CLI] air logs 123 -o json {"type":"ERROR","ts":"[TIMESTAMP]","node":0,"line":"No logs available for run 123. Run terminated in state SUCCESS"} diff --git a/acceptance/air/logs/output.txt b/acceptance/air/logs/output.txt index da16c857ad6..c3d35ed8b3f 100644 --- a/acceptance/air/logs/output.txt +++ b/acceptance/air/logs/output.txt @@ -1,11 +1,13 @@ === logs (text, completed run) + >>> [CLI] air logs 123 step 1 step 2 CUDA out of memory === logs (json) + >>> [CLI] air logs 123 -o json {"type":"LOG","ts":"[TIMESTAMP]","node":0,"line":"step 1"} {"type":"LOG","ts":"[TIMESTAMP]","node":0,"line":"step 2"} @@ -13,34 +15,40 @@ CUDA out of memory {"type":"LOG","ts":"[TIMESTAMP]","node":0,"line":"CUDA out of memory"} === logs with --minutes + >>> [CLI] air logs 123 --minutes 30 step 1 step 2 CUDA out of memory === logs with --tail + >>> [CLI] air logs 123 --tail 1 CUDA out of memory === logs with --tail 0 is rejected + >>> [CLI] air logs 123 --tail 0 Error: invalid --tail 0: must be positive Exit code: 1 === logs from a specific retry + >>> [CLI] air logs 123 --retry 0 step 1 step 2 CUDA out of memory === logs --tail and --minutes are mutually exclusive + >>> [CLI] air logs 123 --tail 100 --minutes 30 Error: cannot combine --tail with --minutes: --tail selects by line count, --minutes by time window Exit code: 1 === logs --tail and --minutes are mutually exclusive (json) + >>> [CLI] air logs 123 --tail 100 --minutes 30 -o json { "v": 1, @@ -56,12 +64,14 @@ Exit code: 1 Exit code: 1 === invalid run id + >>> [CLI] air logs notanumber Error: invalid JOB_RUN_ID "notanumber": must be a positive integer Exit code: 1 === negative node + >>> [CLI] air logs 123 --node -1 Error: invalid --node -1: must not be negative diff --git a/acceptance/air/pools/output.txt b/acceptance/air/pools/output.txt index d2064b55703..834510bbd3c 100644 --- a/acceptance/air/pools/output.txt +++ b/acceptance/air/pools/output.txt @@ -1,11 +1,13 @@ === list pools (text) + >>> [CLI] air pools list ID ACCELERATOR RESERVED pool-8xh100-alpha GPU_8xH100 64 pool-1xh100-beta GPU_1xH100 8 === list pools (json) + >>> [CLI] air pools list -o json { "v": 1, @@ -27,6 +29,7 @@ pool-1xh100-beta GPU_1xH100 8 } === get pool (text) + >>> [CLI] air pools get pool-8xh100-alpha Pool ID: pool-8xh100-alpha Accelerator Type: GPU_8xH100 @@ -35,6 +38,7 @@ Used Accelerators: 40 Idle Accelerators: 24 === get pool (json) + >>> [CLI] air pools get pool-8xh100-alpha -o json { "v": 1, diff --git a/acceptance/air/run-backend-error/output.txt b/acceptance/air/run-backend-error/output.txt index 43871d8188c..bc9d2188d01 100644 --- a/acceptance/air/run-backend-error/output.txt +++ b/acceptance/air/run-backend-error/output.txt @@ -1,5 +1,6 @@ === backend field errors fail dry-run + >>> [CLI] air run -f valid.yaml --dry-run Error: config validation failed: deployments[0].compute.accelerator_count: accelerator_count must be at most 1024, but was 1032. diff --git a/acceptance/air/run-submit-databricks-ai/output.txt b/acceptance/air/run-submit-databricks-ai/output.txt index dbc134d186b..0635ef3d307 100644 --- a/acceptance/air/run-submit-databricks-ai/output.txt +++ b/acceptance/air/run-submit-databricks-ai/output.txt @@ -1,5 +1,6 @@ === submit with a Databricks AI environment and no dependencies + >>> [CLI] air run -f run.yaml Submitting experiment: databricks-ai-v5-smoke Submitted workload with Job Run ID: 555 @@ -10,6 +11,7 @@ Stream logs after submission using: databricks air logs 555 === submit includes the Databricks AI environment version and omits dependencies + >>> print_requests.py //api/2.2/jobs/runs/submit { "method": "POST", diff --git a/acceptance/air/run-submit-deps/output.txt b/acceptance/air/run-submit-deps/output.txt index 3346fcd06d7..ffe886f9b6a 100644 --- a/acceptance/air/run-submit-deps/output.txt +++ b/acceptance/air/run-submit-deps/output.txt @@ -1,5 +1,6 @@ === submit with inline dependencies + >>> [CLI] air run -f run.yaml Submitting experiment: deps-smoke Submitted workload with Job Run ID: 555 @@ -10,11 +11,13 @@ Stream logs after submission using: databricks air logs 555 === only config + command are uploaded; no requirements.yaml + >>> print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique --keep {"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/command.sh", "q": {"overwrite": "true"}, "raw_body": "python train.py"} {"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/deps-smoke/deps-smoke_[RUN_ID]/training_config.yaml", "q": {"overwrite": "true"}, "raw_body": "experiment_name: deps-smoke\ncommand: python train.py\ncompute:\n accelerator_type: GPU_1xH100\n num_accelerators: 1\n pool_id: capacity-123\nenvironment:\n version: 5\n dependencies:\n - numpy\n - torch==2.3.0\nmlflow_artifact_location: /Volumes/main/default/air-artifacts\n"} === inline deps, artifact location, and capacity id reach the submit payload + >>> print_requests.py //api/2.2/jobs/runs/submit { "method": "POST", @@ -61,6 +64,7 @@ Stream logs after submission using: } === a requirements.yaml file path is rejected; deps must be inline + >>> [CLI] air run -f run-file.yaml Error: invalid config run-file.yaml: environment.dependencies must be a list of packages or reference a requirements.txt (see https://docs.databricks.com/aws/en/machine-learning/ai-runtime/cli/yaml-config#reference). A direct file reference is not supported diff --git a/acceptance/air/run-submit-overrides/output.txt b/acceptance/air/run-submit-overrides/output.txt index 0be49262d1b..f3acef3852d 100644 --- a/acceptance/air/run-submit-overrides/output.txt +++ b/acceptance/air/run-submit-overrides/output.txt @@ -1,5 +1,6 @@ === submit with nested overrides + >>> [CLI] air run -f run.yaml --override compute.num_accelerators=2 --override parameters.model.hidden_size=2048 --override parameters.optimizer.learning_rate=0.001 Override: changing compute.num_accelerators from 1 to 2 Override: changing parameters.model.hidden_size from 1024 to 2048 @@ -13,6 +14,7 @@ Stream logs after submission using: databricks air logs 555 === the uploaded config preserves source order and contains overrides + >>> print_requests.py //api/2.0/workspace-files/import-file --oneline --sort --unique {"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/command.sh", "q": {"overwrite": "true"}, "raw_body": "python train.py"} {"method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.air/cli_launch/overrides-smoke/overrides-smoke_[RUN_ID]/hyperparameters.yaml", "q": {"overwrite": "true"}, "raw_body": "model:\n hidden_size: 2048\noptimizer:\n learning_rate: 0.001\n name: adamw\n"} diff --git a/acceptance/air/run-submit/output.txt b/acceptance/air/run-submit/output.txt index 1a88a210162..3119c788b51 100644 --- a/acceptance/air/run-submit/output.txt +++ b/acceptance/air/run-submit/output.txt @@ -1,5 +1,6 @@ === submit with a git code_source + >>> [CLI] air run -f run.yaml Submitting experiment: submit-smoke Uploading [SNAPSHOT_TARBALL]... @@ -11,6 +12,7 @@ Stream logs after submission using: databricks air logs 555 === the ai_runtime_task and additive job + experiment permission grants + >>> print_requests.py --get //api/2.2/jobs/runs/submit //api/2.2/jobs/runs/get //api/2.0/permissions //api/2.0/mlflow --sort --unique { "method": "GET", @@ -106,6 +108,7 @@ Stream logs after submission using: } === submit a B300 workload with AI Runtime 6 + >>> [CLI] air run -f b300.yaml Submitting experiment: b300-submit-smoke Submitted workload with Job Run ID: 555 @@ -116,6 +119,7 @@ Stream logs after submission using: databricks air logs 555 === the B300 accelerator and runtime reach the submit payload + >>> print_requests.py //api/2.2/jobs/runs/submit { "method": "POST", @@ -155,6 +159,7 @@ Stream logs after submission using: } === submit an RTX6000 workload with AI Runtime 6 + >>> [CLI] air run -f rtx6000.yaml Submitting experiment: rtx6000-submit-smoke Submitted workload with Job Run ID: 555 @@ -165,6 +170,7 @@ Stream logs after submission using: databricks air logs 555 === the RTX6000 accelerator and runtime reach the submit payload + >>> print_requests.py //api/2.2/jobs/runs/submit { "method": "POST", diff --git a/acceptance/air/run/output.txt b/acceptance/air/run/output.txt index 1f59b17b246..8e71b6bcfd8 100644 --- a/acceptance/air/run/output.txt +++ b/acceptance/air/run/output.txt @@ -1,9 +1,11 @@ === dry-run (text) + >>> [CLI] air run -f valid.yaml --dry-run Dry run: configuration for "smoke-test" is valid; not submitting. === dry-run (json) + >>> [CLI] air run -f valid.yaml --dry-run -o json { "v": 1, @@ -15,12 +17,14 @@ Dry run: configuration for "smoke-test" is valid; not submitting. } === override applies and logs the change + >>> [CLI] air run -f valid.yaml --dry-run --override compute.num_accelerators=2 --override timeout_minutes=45 Override: changing compute.num_accelerators from 1 to 2 Override: setting timeout_minutes to 45 Dry run: configuration for "smoke-test" is valid; not submitting. === override of an unknown field is rejected + >>> [CLI] air run -f valid.yaml --dry-run --override bogus=1 Error: invalid --override "bogus": "bogus" is not a known field; available fields are: code_source, command, compute, containers, env_variables, environment, experiment_name, idempotency_token, max_retries, mlflow_artifact_location, mlflow_experiment_directory, mlflow_run_name, parameters, permissions, secrets, timeout_minutes, usage_policy_id, usage_policy_name @@ -30,6 +34,7 @@ Tip: use the --debug flag to see more details and a trace of this error: Exit code: 1 === override still runs schema validation + >>> [CLI] air run -f valid.yaml --dry-run --override compute.num_accelerators=0 Override: changing compute.num_accelerators from 1 to 0 Error: compute.num_accelerators must be positive, got 0 @@ -40,14 +45,17 @@ Tip: use the --debug flag to see more details and a trace of this error: Exit code: 1 === watch is ignored with dry-run (nothing is submitted) + >>> [CLI] air run -f valid.yaml --dry-run --watch Dry run: configuration for "smoke-test" is valid; not submitting. === code_source config passes validation + >>> [CLI] air run -f with-code-source.yaml --dry-run Dry run: configuration for "smoke-test" is valid; not submitting. === git.remote is rejected + >>> [CLI] air run -f git-remote.yaml --dry-run Error: git.remote is no longer supported: the snapshot archives your local copy, so a branch resolves to its local HEAD. To deploy a specific committed revision, use git.commit @@ -57,6 +65,7 @@ Tip: use the --debug flag to see more details and a trace of this error: Exit code: 1 === invalid config is rejected + >>> [CLI] air run -f invalid.yaml --dry-run Error: invalid experiment_name "bad.name": only alphanumeric characters, hyphens (-), and underscores (_) are allowed @@ -66,12 +75,14 @@ Tip: use the --debug flag to see more details and a trace of this error: Exit code: 1 === missing --file + >>> [CLI] air run --dry-run Error: required flag(s) "file" not set Exit code: 1 === experimental path directs users to the top-level command + >>> [CLI] experimental air run -f valid.yaml --dry-run Error: the AIR commands have moved out of experimental; use "databricks air" instead diff --git a/acceptance/apps/init/auth-mode/output.txt b/acceptance/apps/init/auth-mode/output.txt index 19ee20452c9..075e81ea31b 100644 --- a/acceptance/apps/init/auth-mode/output.txt +++ b/acceptance/apps/init/auth-mode/output.txt @@ -1,5 +1,6 @@ === Default: every resource bound to the service principal + >>> [CLI] apps init --template ./template --skip-install --name app-sp --features analytics,genie,jobs --set analytics.sql-warehouse.id=wh1 --set genie.genie-space.id=g1 --set genie.genie-space.name=gs --set jobs.job.id=42 --set jobs.secret.scope=s --set jobs.secret.key=k ✔ Creating project @@ -86,6 +87,7 @@ targets: secret_scope: s === App-wide OBO, job kept on the service principal + >>> [CLI] apps init --template ./template --skip-install --name app-obo --features analytics,genie,jobs --set analytics.sql-warehouse.id=wh1 --set genie.genie-space.id=g1 --set genie.genie-space.name=gs --set jobs.job.id=42 --set jobs.secret.scope=s --set jobs.secret.key=k --auth-mode obo --set jobs.job.authMode=sp Note: these resources cannot be accessed on behalf of the user and use the service principal: secret ✔ Creating project @@ -157,6 +159,7 @@ targets: secret_scope: s === Mixed: warehouse both, genie OBO + >>> [CLI] apps init --template ./template --skip-install --name app-mixed --features analytics,genie,jobs --set analytics.sql-warehouse.id=wh1 --set genie.genie-space.id=g1 --set genie.genie-space.name=gs --set jobs.job.id=42 --set jobs.secret.scope=s --set jobs.secret.key=k --set analytics.sql-warehouse.authMode=both --set genie.genie-space.authMode=obo ✔ Creating project @@ -234,6 +237,7 @@ targets: secret_scope: s === App-wide OBO: resources without a scope stay on the service principal + >>> [CLI] apps init --template ./template --skip-install --name app-obo-lenient --features analytics,genie,jobs --set analytics.sql-warehouse.id=wh1 --set genie.genie-space.id=g1 --set genie.genie-space.name=gs --set jobs.job.id=42 --set jobs.secret.scope=s --set jobs.secret.key=k --auth-mode obo Note: these resources cannot be accessed on behalf of the user and use the service principal: job, secret ✔ Creating project @@ -305,6 +309,7 @@ targets: secret_scope: s === Analytics OBO, genie space on the service principal: plugin scopes still emitted + >>> [CLI] apps init --template ./template --skip-install --name app-genie-sp --features analytics,genie,jobs --set analytics.sql-warehouse.id=wh1 --set genie.genie-space.id=g1 --set genie.genie-space.name=gs --set jobs.job.id=42 --set jobs.secret.scope=s --set jobs.secret.key=k --set analytics.sql-warehouse.authMode=obo --set genie.genie-space.authMode=sp ✔ Creating project @@ -386,6 +391,7 @@ targets: secret_scope: s === Errors + >>> errcode [CLI] apps init --template ./template --skip-install --name app-err --features jobs --set jobs.job.authMode=obo Error: resource "job" of plugin "jobs" cannot be accessed on behalf of the user; use jobs.job.authMode=sp @@ -402,6 +408,7 @@ Error: invalid --auth-mode value: "both" (must be obo or sp; use --set . Exit code: 1 === Validate: OBO resource whose scope was removed from user_api_scopes + >>> [CLI] apps init --template ./template --skip-install --name app-val --features analytics --set analytics.sql-warehouse.id=wh1 --set analytics.sql-warehouse.authMode=obo ✔ Creating project diff --git a/acceptance/apps/init/binding-values/output.txt b/acceptance/apps/init/binding-values/output.txt index d23b2f64453..d870d6a2a28 100644 --- a/acceptance/apps/init/binding-values/output.txt +++ b/acceptance/apps/init/binding-values/output.txt @@ -1,5 +1,6 @@ === Service principal, genie space without name: fails before writing files + >>> errcode [CLI] apps init --template ./template --skip-install --name app-sp --features genie --set genie.genie-space.id=g123 --auth-mode sp Error: missing value for genie.genie-space.name (needed by the genie_space binding); use --set genie.genie-space.name=value @@ -7,6 +8,7 @@ Exit code: 1 app-sp was not created === Service principal with --set name: the binding variables have values + >>> [CLI] apps init --template ./template --skip-install --name app-sp-name --features analytics,genie --set analytics.sql-warehouse.id=wh1 --set genie.genie-space.id=g123 --set genie.genie-space.name=space ✔ Creating project @@ -73,6 +75,7 @@ Workspace: Validation OK! === On behalf of the user, genie space without name: no binding, nothing to check + >>> [CLI] apps init --template ./template --skip-install --name app-obo --features analytics,genie --set analytics.sql-warehouse.id=wh1 --set genie.genie-space.id=g123 --auth-mode obo ✔ Creating project @@ -115,6 +118,7 @@ Workspace: Validation OK! === Mixed: warehouse on the service principal, genie space on behalf of the user + >>> [CLI] apps init --template ./template --skip-install --name app-mixed --features analytics,genie --set analytics.sql-warehouse.id=wh1 --set genie.genie-space.id=g123 --set genie.genie-space.authMode=obo ✔ Creating project diff --git a/acceptance/apps/validate-package-manager/output.txt b/acceptance/apps/validate-package-manager/output.txt index f4ed7abf012..b1cc63c078f 100644 --- a/acceptance/apps/validate-package-manager/output.txt +++ b/acceptance/apps/validate-package-manager/output.txt @@ -1,5 +1,6 @@ === Warn but continue on conflicting lockfiles + >>> [CLI] apps validate --path conflicting Warn: found conflicting package managers; using npm from package-lock.json and ignoring pnpm-lock.yaml Validating project using npm... @@ -12,6 +13,7 @@ Validating project using npm... ✅ All validation checks passed ([DURATION]) === Warn but continue on unsupported lockfiles + >>> [CLI] apps validate --path unsupported Warn: ignoring unsupported lockfile yarn.lock; apps validation uses npm or pnpm Validating project using npm... @@ -24,6 +26,7 @@ Validating project using npm... ✅ All validation checks passed ([DURATION]) === Declared package manager wins over a pnpm lockfile + >>> [CLI] apps validate --path metadata-npm Warn: found conflicting package managers; using npm from package.json and ignoring pnpm-lock.yaml Validating project using npm... diff --git a/acceptance/apps/validate/output.txt b/acceptance/apps/validate/output.txt index bbcd46b0c91..907f6a1755a 100644 --- a/acceptance/apps/validate/output.txt +++ b/acceptance/apps/validate/output.txt @@ -1,5 +1,6 @@ === Validate the directory selected by --path + >>> withdir app [CLI] apps validate --path .. Validating project using pnpm... ✅ Installing dependencies ([DURATION]) @@ -11,6 +12,7 @@ Validating project using pnpm... ✅ All validation checks passed ([DURATION]) === Skip tests + >>> [CLI] apps validate --skip-tests Validating project using pnpm... ✅ Installing dependencies ([DURATION]) @@ -22,6 +24,7 @@ Validating project using pnpm... ✅ All validation checks passed ([DURATION]) === Report a failed script + >>> VALIDATION_TEST_FAIL=run --if-present test musterr [CLI] apps validate Validating project using pnpm... ✅ Installing dependencies ([DURATION]) @@ -41,6 +44,7 @@ validation stderr Error: validation failed === Deploy validates with the same package manager before uploading + >>> VALIDATION_TEST_FAIL=run --if-present build musterr [CLI] apps deploy Validating project using pnpm... ✅ Installing dependencies ([DURATION]) @@ -59,6 +63,7 @@ validation stderr Error: validation failed - fix errors before deploying === Commands executed in the project directory + >>> cat commands.log pnpm install pnpm run --if-present typegen diff --git a/acceptance/auth/bundle_and_profile/output.txt b/acceptance/auth/bundle_and_profile/output.txt index 98347d64007..cafae8c7338 100644 --- a/acceptance/auth/bundle_and_profile/output.txt +++ b/acceptance/auth/bundle_and_profile/output.txt @@ -1,17 +1,21 @@ === Inside the bundle, no flags + >>> errcode [CLI] current-user me "[USERNAME]" === Inside the bundle, target flags + >>> errcode [CLI] current-user me -t dev "[USERNAME]" === Inside the bundle, target and matching profile + >>> errcode [CLI] current-user me -t dev -p DEFAULT "[USERNAME]" === Inside the bundle, profile flag not matching bundle host. Should use profile from the flag and not the bundle. + >>> errcode [CLI] current-user me -p profile_name Warn: [hostmetadata] failed to fetch host metadata for https://non.existing.subdomain.databricks.test, will skip for 1m0s Error: Get "https://non.existing.subdomain.databricks.test/api/2.0/preview/scim/v2/Me": (redacted) @@ -19,6 +23,7 @@ Error: Get "https://non.existing.subdomain.databricks.test/api/2.0/preview/scim/ Exit code: 1 === Inside the bundle, target and not matching profile + >>> errcode [CLI] current-user me -t dev -p profile_name Error: cannot resolve bundle auth configuration: the host in the profile (https://non.existing.subdomain.databricks.test) doesn’t match the host configured in the bundle ([DATABRICKS_TARGET]). The profile "DEFAULT" has host="[DATABRICKS_TARGET]" that matches host in the bundle. To select it, pass "-p DEFAULT" @@ -26,6 +31,7 @@ Error: cannot resolve bundle auth configuration: the host in the profile (https: Exit code: 1 === Bundle commands load bundle configuration when no flags, validation OK + >>> errcode [CLI] bundle validate Name: test-auth Target: dev @@ -37,6 +43,7 @@ Workspace: Validation OK! === Bundle commands load bundle configuration with -t flag, validation OK + >>> errcode [CLI] bundle validate -t dev Name: test-auth Target: dev @@ -48,6 +55,7 @@ Workspace: Validation OK! === Bundle commands load bundle configuration with -p flag, validation not OK (profile host don't match bundle host) + >>> errcode [CLI] bundle validate -p profile_name Error: cannot resolve bundle auth configuration: the host in the profile (https://non.existing.subdomain.databricks.test) doesn’t match the host configured in the bundle ([DATABRICKS_TARGET]). The profile "DEFAULT" has host="[DATABRICKS_TARGET]" that matches host in the bundle. To select it, pass "-p DEFAULT" @@ -61,6 +69,7 @@ Found 1 error Exit code: 1 === Bundle commands load bundle configuration with -t and -p flag, validation OK (profile host match bundle host) + >>> errcode [CLI] bundle validate -t dev -p DEFAULT Name: test-auth Target: dev @@ -72,6 +81,7 @@ Workspace: Validation OK! === Bundle commands load bundle configuration with -t and -p flag, validation not OK (profile host don't match bundle host) + >>> errcode [CLI] bundle validate -t prod -p DEFAULT Warn: [hostmetadata] failed to fetch host metadata for https://bar.test, will skip for 1m0s Error: cannot resolve bundle auth configuration: the host in the profile ([DATABRICKS_TARGET]) doesn’t match the host configured in the bundle (https://bar.test) @@ -86,9 +96,11 @@ Found 1 error Exit code: 1 === Outside the bundle, no flags + >>> errcode [CLI] current-user me "[USERNAME]" === Outside the bundle, profile flag + >>> errcode [CLI] current-user me -p profile_name "[USERNAME]" diff --git a/acceptance/auth/bundle_default_profile/script b/acceptance/auth/bundle_default_profile/script index 2baefcbe139..7f506d44f1b 100644 --- a/acceptance/auth/bundle_default_profile/script +++ b/acceptance/auth/bundle_default_profile/script @@ -10,11 +10,11 @@ unset DATABRICKS_TOKEN unset DATABRICKS_CONFIG_PROFILE cp databricks.yml.no-host.tmpl databricks.yml -title "Bundle without workspace.host: default_profile is honored\n" +title "Bundle without workspace.host: default_profile is honored" trace $CLI bundle validate -o json | jq '{host: .workspace.host, profile: .workspace.profile}' cp databricks.yml.no-host.tmpl databricks.yml -title "--profile overrides default_profile (negative case)\n" +title "--profile overrides default_profile (negative case)" trace errcode $CLI bundle validate -p other -o json | jq '{host: .workspace.host, profile: .workspace.profile}' # Switch to a bundle that pins workspace.host. The default_profile guard in @@ -22,12 +22,12 @@ trace errcode $CLI bundle validate -p other -o json | jq '{host: .workspace.host # silently route the user to a profile pointing at a different host than the # bundle declares. DATABRICKS_HOST="$host_value" envsubst < databricks.yml.with-host.tmpl > databricks.yml -title "Bundle with workspace.host: default_profile is NOT applied\n" +title "Bundle with workspace.host: default_profile is NOT applied" trace errcode $CLI bundle validate -o json | jq '{host: .workspace.host, profile: .workspace.profile}' # Switch to a bundle that pins workspace.profile but no host. The pinned # profile must win over default_profile — configureProfile's guard skips # default_profile when workspace.profile is already set. cp databricks.yml.with-profile.tmpl databricks.yml -title "Bundle with workspace.profile: pinned profile wins over default_profile\n" +title "Bundle with workspace.profile: pinned profile wins over default_profile" trace errcode $CLI bundle validate -o json | jq '{host: .workspace.host, profile: .workspace.profile}' diff --git a/acceptance/auth/host-metadata-cache/script b/acceptance/auth/host-metadata-cache/script index 31f989aba0d..3bde2ffda42 100644 --- a/acceptance/auth/host-metadata-cache/script +++ b/acceptance/auth/host-metadata-cache/script @@ -6,11 +6,11 @@ export DATABRICKS_CACHE_DIR="$TEST_TMP_DIR/cache" # never fetch or populate the cache. envsubst < "./home/.databrickscfg.tmpl" > "./home/.databrickscfg" -title "First invocation populates the cache\n" +title "First invocation populates the cache" $CLI auth profiles --output json -title "Second invocation should read from the cache\n" +title "Second invocation should read from the cache" $CLI auth profiles --output json -title "Only one /.well-known/databricks-config request recorded\n" +title "Only one /.well-known/databricks-config request recorded" print_requests.py //.well-known/databricks-config --get --sort diff --git a/acceptance/bundle/apps/compute_size/output.txt b/acceptance/bundle/apps/compute_size/output.txt index 9ab604797e6..0c68a826302 100644 --- a/acceptance/bundle/apps/compute_size/output.txt +++ b/acceptance/bundle/apps/compute_size/output.txt @@ -1,5 +1,6 @@ === Deploy bundle + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/app-compute-size-[UNIQUE_NAME]/files... Created apps.my_app @@ -7,6 +8,7 @@ Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Update compute size and redeploy + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.apps.my_app diff --git a/acceptance/bundle/apps/delete_deleting/script b/acceptance/bundle/apps/delete_deleting/script index 14267a32af4..a5e9d1454a5 100644 --- a/acceptance/bundle/apps/delete_deleting/script +++ b/acceptance/bundle/apps/delete_deleting/script @@ -15,7 +15,7 @@ cleanup() { } trap cleanup EXIT -title "Deploy the app\n" +title "Deploy the app" trace $CLI bundle deploy &> LOG.deploy cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null @@ -31,7 +31,7 @@ cat LOG.plan_delete.err | contains.py '!panic' '!internal error' > /dev/null trace $CLI apps delete "app-$UNIQUE_NAME" &> LOG.apps_delete cat LOG.apps_delete | contains.py '!panic' '!internal error' > /dev/null -title "Apply the stale delete plan against the DELETING app\n" +title "Apply the stale delete plan against the DELETING app" trace $CLI bundle deploy --auto-approve --plan plan_delete.json &> LOG.deploy_delete cat LOG.deploy_delete | contains.py '!panic' '!internal error' > /dev/null diff --git a/acceptance/bundle/apps/git_source/output.txt b/acceptance/bundle/apps/git_source/output.txt index 4046ed6e152..dab5a83ef77 100644 --- a/acceptance/bundle/apps/git_source/output.txt +++ b/acceptance/bundle/apps/git_source/output.txt @@ -1,5 +1,6 @@ === Validate bundle configuration + >>> [CLI] bundle validate Name: app-git-source-[UNIQUE_NAME] Target: default @@ -10,12 +11,14 @@ Workspace: Validation OK! === Plan bundle deployment + >>> [CLI] bundle plan create apps.my_app Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged === Deploy bundle + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/app-git-source-[UNIQUE_NAME]/files... Created apps.my_app @@ -33,6 +36,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged You can access the app at [APP_NAME]-123.cloud.databricksapps.com === Get app details and verify git_source configuration + >>> [CLI] bundle summary --output json >>> [CLI] apps get [APP_NAME] --output json @@ -47,10 +51,12 @@ You can access the app at [APP_NAME]-123.cloud.databricksapps.com } === Verify no drift after deployment + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Run the app to verify it works + >>> cat tmp.app-run ✓ Getting the status of the app [APP_NAME] ✓ App is in RUNNING state @@ -59,12 +65,14 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged You can access the app at [APP_NAME]-123.cloud.databricksapps.com === Update git_source branch and redeploy + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/app-git-source-[UNIQUE_NAME]/files... Files: 3 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged === Verify config update was applied + >>> [CLI] apps get [APP_NAME] --output json { "name": "[APP_NAME]", diff --git a/acceptance/bundle/apps/job_permissions/output.txt b/acceptance/bundle/apps/job_permissions/output.txt index db0a6cafed3..8275a74383d 100644 --- a/acceptance/bundle/apps/job_permissions/output.txt +++ b/acceptance/bundle/apps/job_permissions/output.txt @@ -16,6 +16,7 @@ Add the following section to the jobs permissions: === After second deploy + >>> [CLI] bundle deploy -qq Warning: app "my_app" references jobs "my_job" which has permissions set. To prevent permission override after deploying the app, please add the app service principal to the jobs permissions at resources.apps.my_app @@ -36,6 +37,7 @@ Add the following section to the jobs permissions: false === Apply fix and redeploy + >>> [CLI] bundle deploy -qq >>> has_manage_run diff --git a/acceptance/bundle/apps/source_code_path_update/output.txt b/acceptance/bundle/apps/source_code_path_update/output.txt index 1998e9489ca..cd7cb48aecc 100644 --- a/acceptance/bundle/apps/source_code_path_update/output.txt +++ b/acceptance/bundle/apps/source_code_path_update/output.txt @@ -1,5 +1,6 @@ === Initial deploy — lifecycle.started starts compute and creates active deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/app-scp-update-[UNIQUE_NAME]/files... Created apps.my_app @@ -7,7 +8,9 @@ Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Run app to redeploy (reproduces the issue precondition of having an active deployment) + === Redeploy from saved plan after description change + >>> [CLI] bundle deploy --plan plan.json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/app-scp-update-[UNIQUE_NAME]/files... Updated apps.my_app diff --git a/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt b/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt index 70bbb6eb108..21724b765ea 100644 --- a/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt +++ b/acceptance/bundle/artifacts/ai_runtime_code_source/output.txt @@ -1,5 +1,6 @@ === Expecting code_source_path rewritten to the uploaded artifact remote path + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[].ai_runtime_task [OUTPUT_DIR]/out.requests.txt { "code_source_path": "/Workspace/foo/bar/artifacts/.internal/code.tgz", @@ -16,5 +17,6 @@ } === Expecting the code_source tarball to be uploaded + >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/code.tgz" diff --git a/acceptance/bundle/artifacts/artifact_upload_for_volumes/output.txt b/acceptance/bundle/artifacts/artifact_upload_for_volumes/output.txt index 43f98e80a8c..7e99691efef 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_volumes/output.txt +++ b/acceptance/bundle/artifacts/artifact_upload_for_volumes/output.txt @@ -5,6 +5,7 @@ Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Expecting 2 wheels in libraries section in /jobs/create + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks [OUTPUT_DIR]/out.requests.txt [ { @@ -48,6 +49,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting environment dependencies to be updated + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments [OUTPUT_DIR]/out.requests.txt [ { @@ -63,6 +65,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting wheel to be uploaded + >>> jq -s .[] | select(.path=="/api/2.0/fs/files/Volumes/foo/bar/artifacts/.internal/source.whl") [OUTPUT_DIR]/out.requests.txt { "method": "PUT", diff --git a/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt b/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt index 2a10679b430..415a1bcd9ae 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt +++ b/acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt @@ -5,6 +5,7 @@ Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Expecting 2 wheels in libraries section in /jobs/create + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks [OUTPUT_DIR]/out.requests.txt [ { @@ -48,11 +49,13 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting wheel to be uploaded + >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl" "/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source.whl" === Expecting environment dependencies to be updated + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments [OUTPUT_DIR]/out.requests.txt [ { @@ -68,6 +71,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting delete request to artifact_path/.internal folder + >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) [OUTPUT_DIR]/out.requests.txt { "method": "POST", @@ -79,6 +83,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Expecting mkdirs request to artifact_path/.internal folder + >>> jq -s .[] | select(.path=="/api/2.0/workspace/mkdirs") | select(.body.path | test(".*/artifacts/.internal")) [OUTPUT_DIR]/out.requests.txt { "method": "POST", diff --git a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt index 2419cc5afbc..99a89e2719d 100644 --- a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt +++ b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt @@ -4,11 +4,13 @@ Files: 4 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 0 unchanged === Expecting wheel to be uploaded + >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl" "/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source.whl" === Expecting delete request to artifact_path/.internal folder + >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) [OUTPUT_DIR]/out.requests.txt { "method": "POST", @@ -20,6 +22,7 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged } === Expecting mkdirs request to artifact_path/.internal folder + >>> jq -s .[] | select(.path=="/api/2.0/workspace/mkdirs") | select(.body.path | test(".*/artifacts/.internal")) [OUTPUT_DIR]/out.requests.txt { "method": "POST", diff --git a/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt b/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt index ee40963cbaa..5eac7fab7cf 100644 --- a/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt +++ b/acceptance/bundle/artifacts/upload_multiple_libraries/output.txt @@ -8,6 +8,7 @@ Files: 7 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Expecting 5 wheels in libraries section in /jobs/create + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks [OUTPUT_DIR]/out.requests.txt [ { @@ -38,6 +39,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting 4 wheels to be uploaded + >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source1.whl" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source2.whl" @@ -49,6 +51,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source4.whl" === Expecting environment dependencies to be updated + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments [OUTPUT_DIR]/out.requests.txt [ { diff --git a/acceptance/bundle/artifacts/whl_change_version/output.txt b/acceptance/bundle/artifacts/whl_change_version/output.txt index 34b272c0f08..7af75a2e3f7 100644 --- a/acceptance/bundle/artifacts/whl_change_version/output.txt +++ b/acceptance/bundle/artifacts/whl_change_version/output.txt @@ -11,6 +11,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged dist/my_test_code-0.1.0-py3-none-any.whl === Expecting 1 wheel in libraries section in /jobs/create + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks [OUTPUT_DIR]/out.requests.txt [ { @@ -29,6 +30,7 @@ dist/my_test_code-0.1.0-py3-none-any.whl ] === Expecting 1 wheel to be uploaded + >>> jq .path [OUTPUT_DIR]/out.requests.txt "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/.gitignore" @@ -58,10 +60,12 @@ dist/my_test_code-0.1.0-py3-none-any.whl dist/my_test_code-0.2.0-py3-none-any.whl === Expecting 1 wheel in libraries section in /jobs/reset + >>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks [OUTPUT_DIR]/out.requests.txt json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl"; === Expecting 1 wheel to be uploaded + >>> jq .path [OUTPUT_DIR]/out.requests.txt "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.2.0-py3-none-any.whl" @@ -72,6 +76,7 @@ json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/defa "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME" === Restore config to target old wheel + >>> update_file.py databricks.yml ./dist/*.whl ./dist/my*0.1.0*.whl >>> [CLI] bundle deploy @@ -88,10 +93,12 @@ dist/my_test_code-0.1.0-py3-none-any.whl dist/my_test_code-0.2.0-py3-none-any.whl === Expecting 1 wheel in libraries section in /jobs/reset + >>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks [OUTPUT_DIR]/out.requests.txt json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl"; === Expecting 1 wheel to be uploaded + >>> jq .path [OUTPUT_DIR]/out.requests.txt "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.1.0-py3-none-any.whl" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.2.0-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_dbfs/output.txt b/acceptance/bundle/artifacts/whl_dbfs/output.txt index b4974a6f5ca..4cc89273c37 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/output.txt +++ b/acceptance/bundle/artifacts/whl_dbfs/output.txt @@ -4,9 +4,11 @@ Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Expecting to find no wheels + >>> errcode find.py --expect 0 whl === Expecting 1 wheel in libraries section in /jobs/create + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks [OUTPUT_DIR]/out.requests.txt [ { @@ -25,6 +27,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting no wheels to be uploaded + >>> errcode sh -c jq .path < "$OUT_REQUESTS" | grep import | grep whl Exit code: 1 diff --git a/acceptance/bundle/artifacts/whl_dynamic/output.txt b/acceptance/bundle/artifacts/whl_dynamic/output.txt index 51be24e7e43..d8127cebdfb 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/output.txt +++ b/acceptance/bundle/artifacts/whl_dynamic/output.txt @@ -25,6 +25,7 @@ } === There are 2 original wheels and 2 patched ones + >>> find.py --expect 4 whl .databricks/bundle/default/patched_wheels/my_prebuilt_whl_other_test_code/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl .databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl @@ -32,6 +33,7 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl prebuilt/other_test_code-0.0.1-py3-none-any.whl === Verify contents of the zip file + >>> find.py --expect 1 .databricks/.*my_test_code.*whl src/__init__.py src/__main__.py @@ -42,12 +44,14 @@ my_test_code-0.0.1+[UNIX_TIME_NANOS][1].dist-info/top_level.txt my_test_code-0.0.1+[UNIX_TIME_NANOS][1].dist-info/RECORD === Expecting 2 patched wheels in libraries section in /jobs/create + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks [OUTPUT_DIR]/out.requests.txt json[1].for_each_task.task.libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl"; json[1].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl"; json[1].libraries[1].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"; === Expecting 2 patched wheels to be uploaded + >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" @@ -57,6 +61,7 @@ json[1].libraries[1].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/defa === Updating the local wheel and deploying again === Verify contents, it should now have new_module.py + >>> find.py --expect 1 .databricks/.*my_test_code.*whl src/__init__.py src/__main__.py @@ -68,12 +73,14 @@ my_test_code-0.0.1+[UNIX_TIME_NANOS][2].dist-info/top_level.txt my_test_code-0.0.1+[UNIX_TIME_NANOS][2].dist-info/RECORD === Expecting 2 patched wheels in libraries section in /jobs/reset + >>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | .body.new_settings.tasks [OUTPUT_DIR]/out.requests.txt json[1].for_each_task.task.libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"; json[1].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"; json[1].libraries[1].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"; === Expecting 2 pached wheels to be uploaded (Bad: it is currently uploaded twice) + >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_dynamic/script b/acceptance/bundle/artifacts/whl_dynamic/script index b73202a65ec..47c03e80fc5 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/script +++ b/acceptance/bundle/artifacts/whl_dynamic/script @@ -26,7 +26,7 @@ trace jq .path < "$OUT_REQUESTS" | grep import | grep whl | sort rm "$OUT_REQUESTS" -title "Updating the local wheel and deploying again\n" +title "Updating the local wheel and deploying again" touch my_test_code/src/new_module.py $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan nostamp < tmp.plan.json > out.plan_update.json diff --git a/acceptance/bundle/artifacts/whl_explicit/output.txt b/acceptance/bundle/artifacts/whl_explicit/output.txt index 55aa3ff229a..494a6dd487f 100644 --- a/acceptance/bundle/artifacts/whl_explicit/output.txt +++ b/acceptance/bundle/artifacts/whl_explicit/output.txt @@ -3,6 +3,7 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheel in libraries section in /jobs/create + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks [OUTPUT_DIR]/out.requests.txt [ { @@ -21,11 +22,13 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheel to be uploaded + >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" === Expecting delete request to artifact_path/.internal folder + >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) [OUTPUT_DIR]/out.requests.txt { "method": "POST", @@ -37,6 +40,7 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl } === Expecting mkdirs request to artifact_path/.internal folder + >>> jq -s .[] | select(.path=="/api/2.0/workspace/mkdirs") | select(.body.path | test(".*/artifacts/.internal")) [OUTPUT_DIR]/out.requests.txt { "method": "POST", diff --git a/acceptance/bundle/artifacts/whl_implicit/output.txt b/acceptance/bundle/artifacts/whl_implicit/output.txt index d589ebc5d74..2858dd53801 100644 --- a/acceptance/bundle/artifacts/whl_implicit/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit/output.txt @@ -3,6 +3,7 @@ dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheels in libraries section in /jobs/create + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks [OUTPUT_DIR]/out.requests.txt [ { @@ -21,6 +22,7 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheels to be uploaded + >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt index 04683b58ffc..70bf62f260b 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt @@ -8,6 +8,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged package/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheel in libraries section in /jobs/create + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body [OUTPUT_DIR]/out.requests.txt { "deployment": { @@ -39,6 +40,7 @@ package/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded + >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/package/my_test_code-0.0.1-py3-none-any.whl" "/api/2.0/workspace-files/import-file/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt index 4172ddf8f51..d168ccc778b 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt @@ -3,6 +3,7 @@ dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheel in libraries section in /jobs/create + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[0].libraries [OUTPUT_DIR]/out.requests.txt [ { @@ -11,6 +12,7 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 1 wheel to be uploaded + >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_multiple/output.txt b/acceptance/bundle/artifacts/whl_multiple/output.txt index 8bf5bef423d..1991901b7a8 100644 --- a/acceptance/bundle/artifacts/whl_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_multiple/output.txt @@ -4,6 +4,7 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl === Expecting 2 wheels in libraries section in /jobs/create + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks [OUTPUT_DIR]/out.requests.txt [ { @@ -25,6 +26,7 @@ my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl ] === Expecting 2 wheels to be uploaded + >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_no_cleanup/output.txt b/acceptance/bundle/artifacts/whl_no_cleanup/output.txt index a61388c76f9..fe02f1ba143 100644 --- a/acceptance/bundle/artifacts/whl_no_cleanup/output.txt +++ b/acceptance/bundle/artifacts/whl_no_cleanup/output.txt @@ -3,14 +3,17 @@ dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheels to be uploaded + >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" === No calls to delete internal folder expected + >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) [OUTPUT_DIR]/out.requests.txt === Expected 2 calls to create artifacts folder (because 2 deploys were done) + >>> jq -s .[] | select(.path=="/api/2.0/workspace/mkdirs") | select(.body.path | test(".*/artifacts/.internal")) [OUTPUT_DIR]/out.requests.txt { "method": "POST", diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt index e43ce01f38c..167db2d012d 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt @@ -10,11 +10,13 @@ Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Expecting to find 2 wheels, same as initially provided + >>> find.py --expect 2 whl dist/lib/other_test_code-0.0.1-py3-none-any.whl dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 2 wheels in libraries section in /jobs/create + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks [OUTPUT_DIR]/out.requests.txt [ { @@ -36,6 +38,7 @@ dist/my_test_code-0.0.1-py3-none-any.whl ] === Expecting 2 wheels to be uploaded + >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt index 9611c9541f2..939e2bff18f 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside/output.txt @@ -8,9 +8,11 @@ Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Expecting to find no wheels in current directory + >>> find.py --expect 0 whl === Expecting 2 wheels in libraries section in /jobs/create + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks [OUTPUT_DIR]/out.requests.txt [ { @@ -32,6 +34,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting 2 wheels to be uploaded + >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt index 324b0a1b18a..29618c0c2f2 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/output.txt @@ -8,11 +8,13 @@ Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Expecting to find two patched wheels in current directory + >>> find.py --expect 2 whl .databricks/bundle/default/patched_wheels/art1_my_test_code/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl .databricks/bundle/default/patched_wheels/art2_other_test_code/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl === Expecting 2 wheels in libraries section in /jobs/create + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.tasks [OUTPUT_DIR]/out.requests.txt [ { @@ -56,6 +58,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Expecting 2 wheels to be uploaded + >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt index eb0d23431f3..e3f148e8b02 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt +++ b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt @@ -3,6 +3,7 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheel in environments section in /jobs/create + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body [OUTPUT_DIR]/out.requests.txt { "deployment": { @@ -41,6 +42,7 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded + >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt b/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt index 39e7ead5144..7b8aae86515 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt +++ b/acceptance/bundle/artifacts/whl_via_environment_key_extras/output.txt @@ -3,6 +3,7 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl === Expecting the environments dependency to keep its [train] extras suffix + >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body [OUTPUT_DIR]/out.requests.txt { "deployment": { @@ -41,5 +42,6 @@ my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl } === Expecting 1 wheel to be uploaded under its bare filename (no extras) + >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/config-remote-sync/cli_defaults/script b/acceptance/bundle/config-remote-sync/cli_defaults/script index cf626b9dc9c..ec7777acbff 100755 --- a/acceptance/bundle/config-remote-sync/cli_defaults/script +++ b/acceptance/bundle/config-remote-sync/cli_defaults/script @@ -13,7 +13,6 @@ job2_id="$(read_id.py job2)" pipeline1_id="$(read_id.py pipeline1)" title "Modify jobs" -echo echo "Change max_concurrent_runs, name, change default tag" edit_resource.py jobs $job1_id <>> [CLI] bundle deploy -t dev Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/dev/files... Created cluster_policies.policy @@ -8,6 +9,7 @@ Files: 3 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Sync the UI edit into shared YAML + >>> [CLI] bundle config-remote-sync -t dev --save Detected changes in 1 resource(s): @@ -28,6 +30,7 @@ Resource: resources.jobs.policy_sync - job_cluster_key: main === Deploy shared YAML with the prod policy + >>> [CLI] bundle deploy -t prod Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/prod/files... Created cluster_policies.policy @@ -36,6 +39,7 @@ Files: 4 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Add a user tag beside the policy tag in the UI + >>> [CLI] bundle config-remote-sync -t dev --save Detected changes in 1 resource(s): @@ -55,6 +59,7 @@ Resource: resources.jobs.policy_sync - task_key: main === Deploy the user tag with the prod policy + >>> [CLI] bundle deploy -t prod Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/prod/files... Updated jobs.policy_sync diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt index dfa52c96c28..eb43efffcc0 100644 --- a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt @@ -29,6 +29,7 @@ Resource: resources.jobs.test_job python_file: ./hello.py === Synced config deploys and converges under the policy + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Updated jobs.test_job @@ -39,6 +40,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === Add a task with user and policy cluster fields in the UI + >>> [CLI] bundle config-remote-sync --save Detected changes in 1 resource(s): @@ -67,6 +69,7 @@ Resource: resources.jobs.test_job + task_key: added_in_ui === Synced config deploys and converges under the policy + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Updated jobs.test_job diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_replaces_array/output.txt b/acceptance/bundle/config-remote-sync/cluster_policy_replaces_array/output.txt index a30a2649e51..75dbda135f1 100644 --- a/acceptance/bundle/config-remote-sync/cluster_policy_replaces_array/output.txt +++ b/acceptance/bundle/config-remote-sync/cluster_policy_replaces_array/output.txt @@ -6,6 +6,7 @@ Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Replace authored scripts with a script from an updated policy + >>> [CLI] cluster-policies edit --json { "policy_id": "[POLICY_ID]", "name": "test-policy-[UNIQUE_NAME]", @@ -19,6 +20,7 @@ No changes detected. >>> diff.py databricks.yml.backup databricks.yml === Authored scripts are kept without copying the policy script + >>> [CLI] bundle validate -o json true diff --git a/acceptance/bundle/config-remote-sync/config_edits/script b/acceptance/bundle/config-remote-sync/config_edits/script index e7defc9b658..49eefb61271 100755 --- a/acceptance/bundle/config-remote-sync/config_edits/script +++ b/acceptance/bundle/config-remote-sync/config_edits/script @@ -12,7 +12,6 @@ $CLI bundle deploy job_id="$(read_id.py my_job)" title "Case 1: Added locally, updated remotely with different value" -echo old=$(cat <<'EOF' email_notifications: on_success: @@ -33,7 +32,6 @@ r["email_notifications"]["on_failure"] = ["remote-failure@example.com"] EOF title "Case 2: Added locally, removed remotely" -echo old=$(cat <<'EOF' max_concurrent_runs: 1 EOF @@ -49,7 +47,6 @@ r.pop("timeout_seconds", None) EOF title "Case 3: Removed locally, removed remotely" -echo old=$(cat <<'EOF' tags: env: dev @@ -69,21 +66,18 @@ r["tags"].pop("version", None) EOF title "Case 4: Updated locally, updated remotely with different value" -echo update_file.py databricks.yml 'max_concurrent_runs: 1' 'max_concurrent_runs: 3' read -r -d '' case4 <<'EOF' || true r["max_concurrent_runs"] = 5 EOF title "Case 5: Updated locally, removed remotely" -echo update_file.py databricks.yml 'env: dev' 'env: config-production' read -r -d '' case5 <<'EOF' || true r["tags"].pop("env", None) EOF title "Case 6: Added locally and remotely with same value (no drift expected)" -echo old=$(cat <<'EOF' my_job: email_notifications: @@ -101,7 +95,6 @@ r["description"] = "A test job" EOF title "Edit job remotely" -echo edit_resource.py jobs $job_id <>> [CLI] bundle deploy -t dev Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/dev/files... Created jobs.policy_sync @@ -8,6 +10,7 @@ Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Sync the UI edit into shared YAML + >>> [CLI] bundle config-remote-sync -t dev --save Detected changes in 1 resource(s): @@ -28,6 +31,7 @@ Resource: resources.jobs.policy_sync - job_cluster_key: main === Deploy shared YAML with the prod policy + >>> [CLI] bundle deploy -t prod Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/prod/files... Created jobs.policy_sync @@ -35,6 +39,7 @@ Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === A deleted policy leaves the job unchanged, including its unrelated edits + >>> [CLI] cluster-policies delete [DEV_POLICY_ID] >>> [CLI] bundle config-remote-sync -t dev --save -o json @@ -53,6 +58,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> diff.py databricks.yml.backup databricks.yml === The job stays skipped on the next sync + >>> [CLI] bundle config-remote-sync -t dev -o json { "status": "partial", @@ -67,6 +73,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Text output reports the incomplete sync + >>> [CLI] bundle config-remote-sync -t dev Sync is incomplete: some remote changes were skipped. Left resources.jobs.policy_sync unchanged: cluster policy [DEV_POLICY_ID] was not found. diff --git a/acceptance/bundle/config-remote-sync/flushed_cache/script b/acceptance/bundle/config-remote-sync/flushed_cache/script index 7571c62bbbc..02b58bf942c 100755 --- a/acceptance/bundle/config-remote-sync/flushed_cache/script +++ b/acceptance/bundle/config-remote-sync/flushed_cache/script @@ -8,11 +8,9 @@ cleanup() { trap cleanup EXIT title "Deploy bundle" -echo $CLI bundle deploy title "Get job ID and modify remote resource" -echo job_id="$(read_id.py test_job)" echo "Modify max_concurrent_runs to 5" edit_resource.py jobs $job_id <&1 || echo "Cache directory removed" title "Check for changes after cache flush" -echo $CLI bundle config-remote-sync diff --git a/acceptance/bundle/config-remote-sync/formatting_preserved/output.txt b/acceptance/bundle/config-remote-sync/formatting_preserved/output.txt index 0a63ba22bac..928a63953b8 100644 --- a/acceptance/bundle/config-remote-sync/formatting_preserved/output.txt +++ b/acceptance/bundle/config-remote-sync/formatting_preserved/output.txt @@ -4,6 +4,7 @@ Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Modify max_concurrent_runs, description, and add timeout + === Detect and save changes Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/formatting_preserved/script b/acceptance/bundle/config-remote-sync/formatting_preserved/script index 526ba1f439a..cc67c62b759 100755 --- a/acceptance/bundle/config-remote-sync/formatting_preserved/script +++ b/acceptance/bundle/config-remote-sync/formatting_preserved/script @@ -19,11 +19,9 @@ r["timeout_seconds"] = 3600 EOF title "Detect and save changes" -echo cp databricks.yml databricks.yml.backup $CLI bundle config-remote-sync --save title "Configuration changes" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/job_fields/script b/acceptance/bundle/config-remote-sync/job_fields/script index 20653434666..27b7c8a3abc 100755 --- a/acceptance/bundle/config-remote-sync/job_fields/script +++ b/acceptance/bundle/config-remote-sync/job_fields/script @@ -17,7 +17,6 @@ job_id="$(read_id.py my_job)" title "Modify job fields remotely" -echo edit_resource.py jobs $job_id <>> [CLI] bundle deploy -qq === Rename b_task, replace a_task notebook_path, add synced_task + === Detect task key rename Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/job_multiple_tasks/script b/acceptance/bundle/config-remote-sync/job_multiple_tasks/script index ff2b7e464ac..fc69b7733b0 100755 --- a/acceptance/bundle/config-remote-sync/job_multiple_tasks/script +++ b/acceptance/bundle/config-remote-sync/job_multiple_tasks/script @@ -52,12 +52,10 @@ r["tasks"].append({ EOF title "Detect and save changes" -echo cp databricks.yml databricks.yml.backup $CLI bundle config-remote-sync --save title "Configuration changes" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup @@ -99,11 +97,9 @@ r["tasks"].append({ EOF title "Detect task key rename" -echo cp databricks.yml databricks.yml.backup2 $CLI bundle config-remote-sync --save title "Configuration changes for task key rename" -echo trace diff.py databricks.yml.backup2 databricks.yml rm databricks.yml.backup2 diff --git a/acceptance/bundle/config-remote-sync/job_params_variables/output.txt b/acceptance/bundle/config-remote-sync/job_params_variables/output.txt index 53434bb9957..72b08b6e94f 100644 --- a/acceptance/bundle/config-remote-sync/job_params_variables/output.txt +++ b/acceptance/bundle/config-remote-sync/job_params_variables/output.txt @@ -4,6 +4,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Add new parameter remotely + === Detect and save changes Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/job_params_variables/script b/acceptance/bundle/config-remote-sync/job_params_variables/script index 8a8acc6d972..0ba81296f02 100644 --- a/acceptance/bundle/config-remote-sync/job_params_variables/script +++ b/acceptance/bundle/config-remote-sync/job_params_variables/script @@ -16,11 +16,9 @@ r["parameters"].append({"name": "region", "default": "us-east-1"}) EOF title "Detect and save changes" -echo cp databricks.yml databricks.yml.backup $CLI bundle config-remote-sync --save title "Configuration changes" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt b/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt index fe64e7fce6f..76e103d9d05 100644 --- a/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt +++ b/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt @@ -9,7 +9,9 @@ Files: 3 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Modify pipeline_task full_refresh to True + === Modify pipeline continuous to True + === Detect and save changes Warning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes. at resources.pipelines.my_pipeline.development diff --git a/acceptance/bundle/config-remote-sync/job_pipeline_task/script b/acceptance/bundle/config-remote-sync/job_pipeline_task/script index d5587a201e1..24560b85db8 100755 --- a/acceptance/bundle/config-remote-sync/job_pipeline_task/script +++ b/acceptance/bundle/config-remote-sync/job_pipeline_task/script @@ -27,11 +27,9 @@ EOF title "Detect and save changes" -echo cp databricks.yml databricks.yml.backup $CLI bundle config-remote-sync --save title "Configuration changes" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/json_status_missing_state/script b/acceptance/bundle/config-remote-sync/json_status_missing_state/script index 8ce345c169f..631957e77a8 100755 --- a/acceptance/bundle/config-remote-sync/json_status_missing_state/script +++ b/acceptance/bundle/config-remote-sync/json_status_missing_state/script @@ -5,5 +5,4 @@ envsubst < databricks.yml.tmpl > databricks.yml # In JSON mode the terraform-removal error is reported inside the payload # (status: failed) with exit 0, not as a hard CLI failure. title "Terraform removal is reported in JSON with exit 0" -echo errcode $CLI bundle config-remote-sync -o json diff --git a/acceptance/bundle/config-remote-sync/json_status_selectors/script b/acceptance/bundle/config-remote-sync/json_status_selectors/script index a375fd79213..17730386ea7 100755 --- a/acceptance/bundle/config-remote-sync/json_status_selectors/script +++ b/acceptance/bundle/config-remote-sync/json_status_selectors/script @@ -15,9 +15,7 @@ job_id="$(read_id.py test_job)" # "status" so the workspace caller can tell a benign no-op apart from an error. title "A selector that matches nothing is reported as skipped, exit 0" -echo errcode $CLI bundle config-remote-sync --select-ids "jobs:no-such-id-999" -o json title "A malformed selector is reported as failed, exit 0" -echo errcode $CLI bundle config-remote-sync --select-ids "no-such-id-999" -o json diff --git a/acceptance/bundle/config-remote-sync/missing_cluster_policy/output.txt b/acceptance/bundle/config-remote-sync/missing_cluster_policy/output.txt index 26b7afc429f..9f323265c43 100644 --- a/acceptance/bundle/config-remote-sync/missing_cluster_policy/output.txt +++ b/acceptance/bundle/config-remote-sync/missing_cluster_policy/output.txt @@ -12,6 +12,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] cluster-policies delete [POLICY_ID] === Selecting the unaffected job does not read policies for unselected jobs + >>> [CLI] bundle config-remote-sync --select-ids jobs:[UNAFFECTED_ID] -o json { "status": "success", @@ -29,6 +30,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //policies/clusters/get --get --oneline === A selected job's policy read error leaves the job unchanged without saving + >>> [CLI] bundle config-remote-sync --select-ids jobs:[ARRAY_CHANGE_ID] --save -o json { "status": "partial", @@ -49,6 +51,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged {"method": "GET", "path": "/api/2.0/policies/clusters/get", "q": {"policy_id": "[POLICY_ID]"}} === An array replacement leaves the selected job unchanged + >>> [CLI] bundle config-remote-sync --select-ids jobs:[ARRAY_CHANGE_ID] --save -o json { "status": "partial", @@ -65,6 +68,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged >>> diff.py databricks.yml.backup databricks.yml === A log-storage switch leaves the selected job unchanged + >>> [CLI] bundle config-remote-sync --select-ids jobs:[LOG_STORAGE_CHANGE_ID] --save -o json { "status": "partial", @@ -81,6 +85,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged >>> diff.py databricks.yml.backup databricks.yml === Switching autoscale to a fixed worker count leaves the selected job unchanged + >>> [CLI] bundle config-remote-sync --select-ids jobs:[WORKER_COUNT_CHANGE_ID] --save -o json { "status": "partial", @@ -97,6 +102,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged >>> diff.py databricks.yml.backup databricks.yml === Unrelated jobs still sync when other jobs depend on the missing policy + >>> [CLI] bundle config-remote-sync --save -o json { "status": "partial", diff --git a/acceptance/bundle/config-remote-sync/multiple_files/output.txt b/acceptance/bundle/config-remote-sync/multiple_files/output.txt index a8d9965bb8c..9532d26fc06 100644 --- a/acceptance/bundle/config-remote-sync/multiple_files/output.txt +++ b/acceptance/bundle/config-remote-sync/multiple_files/output.txt @@ -5,9 +5,13 @@ Files: 8 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Modify job_one: max_concurrent_runs, rename c_task + === Add synced_task to job_one (notebook in sync root, resource in subdirectory) + === Modify job_two: max_concurrent_runs, rename first task, add extra_task + === Add extra_task to local config (not in saved state, triggers entity-level replace) + === Detect and save changes Detected changes in 2 resource(s): diff --git a/acceptance/bundle/config-remote-sync/multiple_files/script b/acceptance/bundle/config-remote-sync/multiple_files/script index 4316a3f1cc4..47d3ce29e66 100755 --- a/acceptance/bundle/config-remote-sync/multiple_files/script +++ b/acceptance/bundle/config-remote-sync/multiple_files/script @@ -65,16 +65,13 @@ cat >> resources/job2.yml << 'YAML' YAML title "Detect and save changes" -echo cp resources/job1.yml resources/job1.yml.backup cp resources/job2.yml resources/job2.yml.backup $CLI bundle config-remote-sync --save title "Changes in job1.yml" -echo trace diff.py resources/job1.yml.backup resources/job1.yml title "Changes in job2.yml" -echo trace diff.py resources/job2.yml.backup resources/job2.yml rm resources/job1.yml.backup resources/job2.yml.backup diff --git a/acceptance/bundle/config-remote-sync/multiple_resources/output.txt b/acceptance/bundle/config-remote-sync/multiple_resources/output.txt index 4cf758aafc7..5baad482916 100644 --- a/acceptance/bundle/config-remote-sync/multiple_resources/output.txt +++ b/acceptance/bundle/config-remote-sync/multiple_resources/output.txt @@ -10,6 +10,7 @@ Files: 3 uploaded, 0 deleted Resources: 7 created, 0 changed, 0 deleted, 0 unchanged === Modify both jobs + === Detect and save changes Detected changes in 2 resource(s): diff --git a/acceptance/bundle/config-remote-sync/multiple_resources/script b/acceptance/bundle/config-remote-sync/multiple_resources/script index 7a89c168d69..107550bf997 100755 --- a/acceptance/bundle/config-remote-sync/multiple_resources/script +++ b/acceptance/bundle/config-remote-sync/multiple_resources/script @@ -33,11 +33,9 @@ r.setdefault("tags", {})["team"] = "ml" EOF title "Detect and save changes" -echo cp databricks.yml databricks.yml.backup $CLI bundle config-remote-sync --save title "Configuration changes" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/output_json/output.txt b/acceptance/bundle/config-remote-sync/output_json/output.txt index 6cf6c7b6b05..bc4250b673c 100644 --- a/acceptance/bundle/config-remote-sync/output_json/output.txt +++ b/acceptance/bundle/config-remote-sync/output_json/output.txt @@ -3,7 +3,8 @@ Created jobs.test_job Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -=== JSON output format{ +=== JSON output format +{ "status": "success", "files": [ { diff --git a/acceptance/bundle/config-remote-sync/output_no_changes/script b/acceptance/bundle/config-remote-sync/output_no_changes/script index 1bef0ab9c02..e30a871d2ff 100755 --- a/acceptance/bundle/config-remote-sync/output_no_changes/script +++ b/acceptance/bundle/config-remote-sync/output_no_changes/script @@ -11,14 +11,11 @@ touch dummy.whl $CLI bundle deploy title "Check for changes immediately after deployment" -echo $CLI bundle config-remote-sync title "Text output" -echo $CLI bundle config-remote-sync | contains.py "No changes detected" title "JSON output" -echo $CLI bundle config-remote-sync -o json > tmp.json cat tmp.json diff --git a/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt b/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt index 968f42c400b..2801092cc88 100644 --- a/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt +++ b/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt @@ -4,6 +4,7 @@ Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Modify pipeline fields remotely + === Detect and save changes Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/pipeline_fields/script b/acceptance/bundle/config-remote-sync/pipeline_fields/script index 1fe0261689b..ec505182dd4 100755 --- a/acceptance/bundle/config-remote-sync/pipeline_fields/script +++ b/acceptance/bundle/config-remote-sync/pipeline_fields/script @@ -42,11 +42,9 @@ EOF # rm permissions.json title "Detect and save changes" -echo cp databricks.yml databricks.yml.backup $CLI bundle config-remote-sync --save title "Configuration changes" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/output.txt b/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/output.txt index 9e6e395dc2b..c1936c71fa7 100644 --- a/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/output.txt +++ b/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/output.txt @@ -5,6 +5,7 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Policy values alone do not need to be synced + >>> [CLI] bundle config-remote-sync No changes detected. @@ -35,6 +36,7 @@ Resource: resources.jobs.job1 - job_cluster_key: shared === Synced config deploys and converges under the policy + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/script b/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/script index e9f0558a9ef..883c68c9a81 100644 --- a/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/script +++ b/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/script @@ -14,7 +14,6 @@ title "Policy values alone do not need to be synced" trace $CLI bundle config-remote-sync title "Simulate a cluster policy injecting custom_tags and cluster_log_conf" -echo echo "These fields exist only remotely (the user never set them in config)." echo "A legitimate edit (max_concurrent_runs) is made too, to show real edits still sync." edit_resource.py jobs $job1_id <<'EOF' @@ -37,12 +36,10 @@ for jc in r.get("job_clusters", []): EOF title "Detect and save all changes" -echo cp databricks.yml databricks.yml.backup $CLI bundle config-remote-sync --save title "Configuration changes" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/resolve_variables/output.txt b/acceptance/bundle/config-remote-sync/resolve_variables/output.txt index 842ae93983d..dc5cb05b491 100644 --- a/acceptance/bundle/config-remote-sync/resolve_variables/output.txt +++ b/acceptance/bundle/config-remote-sync/resolve_variables/output.txt @@ -9,7 +9,9 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Modify pipeline environment dependencies + === Add and replace parameters remotely + === Detect and save changes Warning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes. at resources.pipelines.my_pipeline.development diff --git a/acceptance/bundle/config-remote-sync/resolve_variables/script b/acceptance/bundle/config-remote-sync/resolve_variables/script index a3d0edf7468..ebf76b26cd6 100755 --- a/acceptance/bundle/config-remote-sync/resolve_variables/script +++ b/acceptance/bundle/config-remote-sync/resolve_variables/script @@ -121,17 +121,14 @@ for e in r["environments"]: EOF title "Detect and save changes" -echo cp databricks.yml databricks.yml.backup $CLI bundle config-remote-sync --save title "Configuration changes" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup title "Telemetry" -echo # The engine name is dropped because both EnvMatrix variants share this output; # every other counter is engine-agnostic. This run exercises refs_retargeted # (the env param re-targeted to a different variable) and refs_from_siblings diff --git a/acceptance/bundle/config-remote-sync/resource_keys/output.txt b/acceptance/bundle/config-remote-sync/resource_keys/output.txt index e7de6210739..7d21ae4bdda 100644 --- a/acceptance/bundle/config-remote-sync/resource_keys/output.txt +++ b/acceptance/bundle/config-remote-sync/resource_keys/output.txt @@ -2,7 +2,9 @@ >>> [CLI] bundle deploy -qq === Edit job with spaces remotely + === Sync only job with spaces + >>> [CLI] bundle config-remote-sync --select-ids jobs:[JOB WITH SPACES_ID] --save Detected changes in 1 resource(s): @@ -29,11 +31,13 @@ Resource: resources.jobs.job with spaces name: quoted-job === Syncing job with spaces again is idempotent + >>> [CLI] bundle config-remote-sync --select-ids jobs:[JOB WITH SPACES_ID] --save No changes detected. === Add a sibling task in job with spaces + >>> [CLI] bundle config-remote-sync --select-ids jobs:[JOB WITH SPACES_ID] --save Detected changes in 1 resource(s): @@ -55,7 +59,9 @@ Resource: resources.jobs.job with spaces "job's\"quotes": === Edit job's"quotes remotely + === Sync only job's"quotes + >>> [CLI] bundle config-remote-sync --select-ids jobs:[JOB'S\"QUOTES_ID] --save Detected changes in 1 resource(s): @@ -82,11 +88,13 @@ Resource: resources.jobs.job's"quotes targets: === Syncing job's"quotes again is idempotent + >>> [CLI] bundle config-remote-sync --select-ids jobs:[JOB'S\"QUOTES_ID] --save No changes detected. === Add a sibling task in job's"quotes + >>> [CLI] bundle config-remote-sync --select-ids jobs:[JOB'S\"QUOTES_ID] --save Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/select_basic/output.txt b/acceptance/bundle/config-remote-sync/select_basic/output.txt index efc4894ccd5..4b2b02fb7a4 100644 --- a/acceptance/bundle/config-remote-sync/select_basic/output.txt +++ b/acceptance/bundle/config-remote-sync/select_basic/output.txt @@ -5,6 +5,7 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Modify both jobs remotely + === Sync only job_one, selected by its type and deployed resource id Detected changes in 1 resource(s): @@ -72,6 +73,7 @@ Error: no deployed pipelines resource with id [JOB_ONE_ID]; the deployment state Exit code: 1 === A selector without a type is still rejected + >>> [CLI] bundle config-remote-sync --select-ids no-such-id-123 Error: invalid --select-ids value "no-such-id-123", expected : (e.g. jobs:123456789) diff --git a/acceptance/bundle/config-remote-sync/select_basic/script b/acceptance/bundle/config-remote-sync/select_basic/script index a85bb0b46a6..d5265fa045a 100644 --- a/acceptance/bundle/config-remote-sync/select_basic/script +++ b/acceptance/bundle/config-remote-sync/select_basic/script @@ -21,36 +21,29 @@ r["max_concurrent_runs"] = 10 EOF title "Sync only job_one, selected by its type and deployed resource id" -echo cp databricks.yml databricks.yml.backup $CLI bundle config-remote-sync --select-ids "jobs:$job_one_id" --save title "Only job_one is updated; job_two is left untouched" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup title "Selecting job_one again is idempotent" -echo $CLI bundle config-remote-sync --select-ids "jobs:$job_one_id" title "Unfiltered sync still detects the job_two drift (no lost updates)" -echo $CLI bundle config-remote-sync title "A stale selector is skipped, not fatal: job_two still syncs alongside it" -echo cp databricks.yml databricks.yml.backup $CLI bundle config-remote-sync --select-ids "jobs:no-such-id-123,jobs:$job_two_id" --save trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup title "An unknown resource id alone is rejected (no match at all must not silently succeed)" -echo errcode trace $CLI bundle config-remote-sync --select-ids jobs:no-such-id-123 title "An id that exists under a different type matches nothing and is rejected too" -echo errcode trace $CLI bundle config-remote-sync --select-ids "pipelines:$job_one_id" title "A selector without a type is still rejected" diff --git a/acceptance/bundle/config-remote-sync/select_multiple/output.txt b/acceptance/bundle/config-remote-sync/select_multiple/output.txt index eb64a611009..f6280c89d1d 100644 --- a/acceptance/bundle/config-remote-sync/select_multiple/output.txt +++ b/acceptance/bundle/config-remote-sync/select_multiple/output.txt @@ -6,6 +6,7 @@ Files: 2 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Modify all three jobs remotely + === Comma-separated selectors preview two of three resources Detected changes in 2 resource(s): diff --git a/acceptance/bundle/config-remote-sync/select_multiple/script b/acceptance/bundle/config-remote-sync/select_multiple/script index 8eab7443aff..58b158715b2 100644 --- a/acceptance/bundle/config-remote-sync/select_multiple/script +++ b/acceptance/bundle/config-remote-sync/select_multiple/script @@ -20,23 +20,18 @@ EOF done title "Comma-separated selectors preview two of three resources" -echo $CLI bundle config-remote-sync --select-ids "jobs:$job_a_id,jobs:$job_b_id" title "Repeating the same selector dedupes silently" -echo $CLI bundle config-remote-sync --select-ids "jobs:$job_c_id,jobs:$job_c_id" title "Save with repeated --select-ids flags" -echo cp databricks.yml databricks.yml.backup $CLI bundle config-remote-sync --select-ids "jobs:$job_a_id" --select-ids "jobs:$job_b_id" --save title "job_a and job_b are updated, job_c is untouched" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup title "Unfiltered sync still detects the job_c drift" -echo $CLI bundle config-remote-sync diff --git a/acceptance/bundle/config-remote-sync/skip_deployment/script b/acceptance/bundle/config-remote-sync/skip_deployment/script index 532a889fe76..7fc92ce939c 100755 --- a/acceptance/bundle/config-remote-sync/skip_deployment/script +++ b/acceptance/bundle/config-remote-sync/skip_deployment/script @@ -15,7 +15,6 @@ pipeline_id="$(read_id.py my_pipeline)" title "Modify the CLI-managed deployment metadata and one user field remotely" -echo # deployment.* is rewritten by the CLI on every deploy (metadata.AnnotateJobs / # AnnotatePipelines), so config-remote-sync must never sync it back, even when it # differs from the local value. The description / configuration edits are the @@ -32,11 +31,9 @@ r["configuration"]["key2"] = "value2" EOF title "Detect and save changes" -echo cp databricks.yml databricks.yml.backup $CLI bundle config-remote-sync --save title "Configuration changes" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/skip_permissions/script b/acceptance/bundle/config-remote-sync/skip_permissions/script index 585db4a5535..49b211bf777 100644 --- a/acceptance/bundle/config-remote-sync/skip_permissions/script +++ b/acceptance/bundle/config-remote-sync/skip_permissions/script @@ -11,24 +11,20 @@ $CLI bundle deploy job_id="$(read_id.py my_job)" title "Add a permission remotely (a change on the permissions sub-resource)" -echo # permissions set replaces the whole ACL, so keep the deploy identity as owner # and add a grantee (the built-in "users" group) that is not in config: this # makes the permissions sub-resource show a change on the next sync. $CLI permissions set jobs $job_id --json '{"access_control_list":[{"permission_level":"IS_OWNER","user_name":"'"$CURRENT_USER_NAME"'"},{"permission_level":"CAN_VIEW","group_name":"users"}]}' > /dev/null title "Modify a regular job field remotely" -echo edit_resource.py jobs $job_id <>> grep -c existing_cluster_id databricks.yml 1 === Edit that locationless field remotely on a two-block task + === Sync Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/split/cli_default_split_element/script b/acceptance/bundle/config-remote-sync/split/cli_default_split_element/script index a691d865e98..4c61312b7d1 100644 --- a/acceptance/bundle/config-remote-sync/split/cli_default_split_element/script +++ b/acceptance/bundle/config-remote-sync/split/cli_default_split_element/script @@ -16,11 +16,9 @@ for task in r["tasks"]: PYEDIT title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --save title "The field is added to the block declaring the resource" -echo trace diff.py databricks.yml.backup databricks.yml errcode trace grep -c "existing_cluster_id: 0101-000000-remote1" databricks.yml rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/split/dotted_target/output.txt b/acceptance/bundle/config-remote-sync/split/dotted_target/output.txt index ca7af9375da..cbf7d39b261 100644 --- a/acceptance/bundle/config-remote-sync/split/dotted_target/output.txt +++ b/acceptance/bundle/config-remote-sync/split/dotted_target/output.txt @@ -4,6 +4,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Edit a field in the override block of a target whose name contains a dot + === Sync Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/split/dotted_target/script b/acceptance/bundle/config-remote-sync/split/dotted_target/script index c3d05534ab2..6ddc898a2ab 100644 --- a/acceptance/bundle/config-remote-sync/split/dotted_target/script +++ b/acceptance/bundle/config-remote-sync/split/dotted_target/script @@ -13,10 +13,8 @@ for task in r["tasks"]: PYEDIT title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev.eu --save title "The override block is updated" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/split/isolation/output.txt b/acceptance/bundle/config-remote-sync/split/isolation/output.txt index f8805d02358..6f7b215b515 100644 --- a/acceptance/bundle/config-remote-sync/split/isolation/output.txt +++ b/acceptance/bundle/config-remote-sync/split/isolation/output.txt @@ -5,6 +5,7 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Rename the two-block task on job_a, and edit job_b in the same run + === Sync Detected changes in 2 resource(s): diff --git a/acceptance/bundle/config-remote-sync/split/isolation/script b/acceptance/bundle/config-remote-sync/split/isolation/script index b1320eed712..2e16744f0ca 100644 --- a/acceptance/bundle/config-remote-sync/split/isolation/script +++ b/acceptance/bundle/config-remote-sync/split/isolation/script @@ -27,12 +27,10 @@ r["max_concurrent_runs"] = 6 EOF title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --save title "job_b is updated, and job_a's rename is written in both blocks" -echo trace diff.py databricks.yml.backup databricks.yml trace grep -c "max_concurrent_runs: 6" databricks.yml trace grep -c "task_key: shared_renamed" databricks.yml diff --git a/acceptance/bundle/config-remote-sync/split/keyed_edit/output.txt b/acceptance/bundle/config-remote-sync/split/keyed_edit/output.txt index 7dd08e2e21c..bef9716a09b 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_edit/output.txt +++ b/acceptance/bundle/config-remote-sync/split/keyed_edit/output.txt @@ -4,6 +4,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Edit alpha, defined only in the target block + === Sync Detected changes in 1 resource(s): @@ -23,6 +24,7 @@ Resource: resources.jobs.split_job + timeout_seconds: 111 === Edit mu, a top-level task that is not first after the merge sort + === Sync Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/split/keyed_edit/script b/acceptance/bundle/config-remote-sync/split/keyed_edit/script index dcfc1028455..dde2c705431 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_edit/script +++ b/acceptance/bundle/config-remote-sync/split/keyed_edit/script @@ -22,12 +22,10 @@ for task in r["tasks"]: EOF title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --save title "Only alpha in the target block gains timeout_seconds: 111" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup @@ -43,11 +41,9 @@ for task in r["tasks"]: EOF title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --save title "Only mu in the top-level block gains timeout_seconds: 222" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/split/keyed_remove/output.txt b/acceptance/bundle/config-remote-sync/split/keyed_remove/output.txt index 4a9d9bb6657..0f421f5df6f 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_remove/output.txt +++ b/acceptance/bundle/config-remote-sync/split/keyed_remove/output.txt @@ -5,6 +5,7 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Remove gamma from the top-level block and alpha from the target block + === Sync Detected changes in 1 resource(s): @@ -39,6 +40,7 @@ Resource: resources.jobs.remove_job tasks: === Remove the task defined in BOTH blocks + === Sync Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/split/keyed_remove/script b/acceptance/bundle/config-remote-sync/split/keyed_remove/script index 73998fa670b..19a7b022d38 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_remove/script +++ b/acceptance/bundle/config-remote-sync/split/keyed_remove/script @@ -21,12 +21,10 @@ r["tasks"] = [t for t in r["tasks"] if t["task_key"] not in ("gamma", "alpha")] EOF title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --select-ids "jobs:$remove_job_id" --save title "Exactly gamma and alpha are gone; beta survives" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup @@ -40,12 +38,10 @@ r["tasks"] = [t for t in r["tasks"] if t["task_key"] != "both"] EOF title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --select-ids "jobs:$twoblock_job_id" --save title "'both' is gone from both blocks; 'keep' survives" -echo trace diff.py databricks.yml.backup databricks.yml # grep -c exits non-zero on a count of zero, which would abort the script under # bash -e before the second assertion runs. diff --git a/acceptance/bundle/config-remote-sync/split/keyed_rename/output.txt b/acceptance/bundle/config-remote-sync/split/keyed_rename/output.txt index 80194ffa904..b77e1c003db 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_rename/output.txt +++ b/acceptance/bundle/config-remote-sync/split/keyed_rename/output.txt @@ -5,6 +5,7 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Rename a task defined in both blocks and a target-only task + === Sync Detected changes in 1 resource(s): @@ -46,6 +47,7 @@ Resource: resources.jobs.rename_job 1 === Remove a task and rename a later one in the same run, single block + === Sync Detected changes in 1 resource(s): @@ -75,6 +77,7 @@ Resource: resources.jobs.shift_job 1 === Rename the two-block task AND edit one of its fields in the same run + === Sync Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/split/keyed_rename/script b/acceptance/bundle/config-remote-sync/split/keyed_rename/script index 88b8106ed55..85969586574 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_rename/script +++ b/acceptance/bundle/config-remote-sync/split/keyed_rename/script @@ -26,12 +26,10 @@ for task in r["tasks"]: EOF title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --save title "Both blocks get the new key; each field keeps its own scope" -echo trace diff.py databricks.yml.backup databricks.yml errcode trace grep -c "task_key: a_shared" databricks.yml errcode trace grep -c "task_key: z_solo" databricks.yml @@ -50,12 +48,10 @@ for task in r["tasks"]: EOF title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --select-ids "jobs:$shift_job_id" --save title "aaa is gone, mmm became mmm2, zzz keeps its name and its max_retries" -echo trace diff.py databricks.yml.backup databricks.yml errcode trace grep -c "task_key: zzz" databricks.yml rm databricks.yml.backup @@ -74,12 +70,10 @@ for task in r["tasks"]: EOF title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --select-ids "jobs:$job_id" --save title "Left unapplied: the split is intact and timeout_seconds stays target-scoped" -echo trace diff.py databricks.yml.backup databricks.yml errcode trace grep -c "task_key: a_shared" databricks.yml errcode trace grep -c "timeout_seconds: 45" databricks.yml diff --git a/acceptance/bundle/config-remote-sync/split/keyed_twoblock/output.txt b/acceptance/bundle/config-remote-sync/split/keyed_twoblock/output.txt index 6fde29887a5..c66385d2d69 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_twoblock/output.txt +++ b/acceptance/bundle/config-remote-sync/split/keyed_twoblock/output.txt @@ -4,6 +4,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Edit omega, whose sibling is defined in two blocks + === Sync Detected changes in 1 resource(s): @@ -23,6 +24,7 @@ Resource: resources.jobs.twoblock_job + timeout_seconds: 303 === Edit a field per block, a field defined in both, and add a new field + === Sync Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/split/keyed_twoblock/script b/acceptance/bundle/config-remote-sync/split/keyed_twoblock/script index 2b7d8cf82ce..f5f93f19a2d 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_twoblock/script +++ b/acceptance/bundle/config-remote-sync/split/keyed_twoblock/script @@ -22,12 +22,10 @@ for task in r["tasks"]: EOF title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --save title "Only omega changes; shared is untouched in both blocks" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup @@ -47,11 +45,9 @@ for task in r["tasks"]: EOF title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --save title "Each edit lands on the definition that was deployed" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/split/multifile/output.txt b/acceptance/bundle/config-remote-sync/split/multifile/output.txt index de38bd589d1..f6bc8e976ab 100644 --- a/acceptance/bundle/config-remote-sync/split/multifile/output.txt +++ b/acceptance/bundle/config-remote-sync/split/multifile/output.txt @@ -4,6 +4,7 @@ Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Edit a task defined in the second included file + === Sync Detected changes in 1 resource(s): @@ -33,6 +34,7 @@ Resource: resources.jobs.multifile_job timeout_seconds: 30 === Edit one field per file on the task declared in both override files + === Sync Detected changes in 1 resource(s): @@ -67,6 +69,7 @@ Resource: resources.jobs.multifile_job max_retries: 2 === Edit a field that both override files define + === Sync Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/split/multifile/script b/acceptance/bundle/config-remote-sync/split/multifile/script index 4f82441d08e..de15de40b16 100644 --- a/acceptance/bundle/config-remote-sync/split/multifile/script +++ b/acceptance/bundle/config-remote-sync/split/multifile/script @@ -24,22 +24,18 @@ for task in r["tasks"]: EOF title "Sync" -echo cp databricks.yml databricks.yml.backup cp overrides/10-first.yml overrides/10-first.yml.backup cp overrides/20-second.yml overrides/20-second.yml.backup errcode $CLI bundle config-remote-sync -t dev --save title "databricks.yml is untouched" -echo trace diff.py databricks.yml.backup databricks.yml title "overrides/10-first.yml is untouched" -echo trace diff.py overrides/10-first.yml.backup overrides/10-first.yml title "overrides/20-second.yml gains timeout_seconds: 777" -echo trace diff.py overrides/20-second.yml.backup overrides/20-second.yml rm databricks.yml.backup overrides/10-first.yml.backup overrides/20-second.yml.backup @@ -58,13 +54,11 @@ for task in r["tasks"]: EOF title "Sync" -echo cp overrides/10-first.yml overrides/10-first.yml.backup cp overrides/20-second.yml overrides/20-second.yml.backup errcode $CLI bundle config-remote-sync -t dev --save title "max_retries updates the first file, timeout_seconds the second" -echo trace diff.py overrides/10-first.yml.backup overrides/10-first.yml trace diff.py overrides/20-second.yml.backup overrides/20-second.yml rm overrides/10-first.yml.backup overrides/20-second.yml.backup @@ -80,13 +74,11 @@ for task in r["tasks"]: PYEDIT title "Sync" -echo cp overrides/10-first.yml overrides/10-first.yml.backup cp overrides/20-second.yml overrides/20-second.yml.backup errcode $CLI bundle config-remote-sync -t dev --save title "The winning file is updated; the shadowed copy is left alone" -echo trace diff.py overrides/10-first.yml.backup overrides/10-first.yml trace diff.py overrides/20-second.yml.backup overrides/20-second.yml rm overrides/10-first.yml.backup overrides/20-second.yml.backup diff --git a/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/output.txt b/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/output.txt index 5b5a85fab57..fe3f138128b 100644 --- a/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/output.txt +++ b/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/output.txt @@ -4,6 +4,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Add a task, then depend on it from the split task zzz_shared + === Sync Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/script b/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/script index cc23a266e2a..450b53fe199 100644 --- a/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/script +++ b/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/script @@ -15,11 +15,9 @@ for task in r["tasks"]: PYEDIT title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --save title "The dependency lands on zzz_shared, in the block that defines it" -echo trace diff.py databricks.yml.backup databricks.yml errcode trace grep -c "task_key: mid_new" databricks.yml rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/split/nested_sequence/output.txt b/acceptance/bundle/config-remote-sync/split/nested_sequence/output.txt index 0074cfeafce..4cba045aed4 100644 --- a/acceptance/bundle/config-remote-sync/split/nested_sequence/output.txt +++ b/acceptance/bundle/config-remote-sync/split/nested_sequence/output.txt @@ -4,6 +4,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Add a library to a task that is not first after the merge sort + === Sync Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/split/nested_sequence/script b/acceptance/bundle/config-remote-sync/split/nested_sequence/script index 26d26468995..9946459a9e9 100644 --- a/acceptance/bundle/config-remote-sync/split/nested_sequence/script +++ b/acceptance/bundle/config-remote-sync/split/nested_sequence/script @@ -22,12 +22,10 @@ for task in r["tasks"]: PYEDIT title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --save title "The new library lands on the task that received it" -echo trace diff.py databricks.yml.backup databricks.yml errcode trace grep -c "pyyaml" databricks.yml rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/split/positional/output.txt b/acceptance/bundle/config-remote-sync/split/positional/output.txt index 9516f90baa0..d53cd8c3a49 100644 --- a/acceptance/bundle/config-remote-sync/split/positional/output.txt +++ b/acceptance/bundle/config-remote-sync/split/positional/output.txt @@ -4,6 +4,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Edit num_workers on the top-level cluster and on the target cluster + === Sync Detected changes in 1 resource(s): @@ -32,6 +33,7 @@ Resource: resources.pipelines.split_pipeline + num_workers: 3 === Remove the target-block cluster (length change) + === Sync Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/split/positional/script b/acceptance/bundle/config-remote-sync/split/positional/script index fa7b1d54db1..2c7392cf9f2 100644 --- a/acceptance/bundle/config-remote-sync/split/positional/script +++ b/acceptance/bundle/config-remote-sync/split/positional/script @@ -23,12 +23,10 @@ for cluster in r["clusters"]: EOF title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --save title "default updates top-level, maintenance updates the target block" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup @@ -47,12 +45,10 @@ r["clusters"] = [c for c in r["clusters"] if c["label"] != "maintenance"] EOF title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --save title "The top-level cluster is untouched; the target entry is left as a remnant" -echo trace diff.py databricks.yml.backup databricks.yml errcode trace grep -c "label: default" databricks.yml errcode trace grep -c "label: maintenance" databricks.yml diff --git a/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/output.txt b/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/output.txt index 83af06feaba..0e1cc402230 100644 --- a/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/output.txt +++ b/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/output.txt @@ -4,6 +4,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Remove max_retries, which both blocks define + === Sync Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/script b/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/script index 876cb123a51..c12803edb91 100644 --- a/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/script +++ b/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/script @@ -13,11 +13,9 @@ for task in r["tasks"]: PYEDIT title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --save title "Both copies must be gone, else the next deploy pushes the value back" -echo trace diff.py databricks.yml.backup databricks.yml errcode trace grep -c "max_retries" databricks.yml rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/output.txt b/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/output.txt index 0da6b78b7d9..85168a94ebf 100644 --- a/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/output.txt +++ b/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/output.txt @@ -4,6 +4,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Remove the two-block task AND add an unrelated new task in one run + === Sync Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/script b/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/script index 26f73be2078..c3e29f130dd 100644 --- a/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/script +++ b/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/script @@ -15,11 +15,9 @@ r["tasks"].append({"task_key": "brand_new", "max_retries": 9, PYEDIT title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --save title "brand_new is added even though the removal of shared was held back" -echo trace diff.py databricks.yml.backup databricks.yml errcode trace grep -c "task_key: brand_new" databricks.yml errcode trace grep -c "task_key: keep" databricks.yml diff --git a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/output.txt b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/output.txt index de185fb52ca..f88a40eb3c6 100644 --- a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/output.txt +++ b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/output.txt @@ -4,6 +4,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === alpha (top-level) -> zzz and beta (target block) -> aaa, bodies identical + === Sync Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/script b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/script index 4d4d0bc5972..f64fba40450 100644 --- a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/script +++ b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/script @@ -17,11 +17,9 @@ for task in r["tasks"]: PYEDIT title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --save title "Neither rename is applied, and both blocks keep their own task" -echo trace diff.py databricks.yml.backup databricks.yml errcode trace grep -c "task_key: alpha" databricks.yml errcode trace grep -c "task_key: beta" databricks.yml diff --git a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/output.txt b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/output.txt index 284fdae5f88..e2b0de7a174 100644 --- a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/output.txt +++ b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/output.txt @@ -4,6 +4,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Rename alpha -> zzz and beta -> aaa, and point gamma at the new keys + === Sync Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/script b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/script index ffbbfc09eab..f00c30c8d1e 100644 --- a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/script +++ b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/script @@ -19,12 +19,10 @@ for task in r["tasks"]: PYEDIT title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync --save title "Both renames apply and depends_on follows them" -echo trace diff.py databricks.yml.backup databricks.yml echo echo "=== Checking for old task keys (should be gone if renames applied):" diff --git a/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/output.txt b/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/output.txt index 4bec8863fd6..3d6c30dcfb1 100644 --- a/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/output.txt +++ b/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/output.txt @@ -4,6 +4,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Two identical tasks, one in each block; DELETE one (remote), RENAME the other + === Sync Detected changes in 1 resource(s): @@ -20,6 +21,7 @@ Resource: resources.jobs.two_removes_job === Both tasks keep their original keys, and no new key was written + >>> grep -c task_key: task_toplevel databricks.yml 1 diff --git a/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/script b/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/script index 045fec9922e..fd39ffc836c 100644 --- a/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/script +++ b/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/script @@ -22,12 +22,10 @@ r["tasks"] = new_tasks PYEDIT title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --save title "Neither block is rewritten: the pairing is not forced" -echo trace diff.py databricks.yml.backup databricks.yml echo title "Both tasks keep their original keys, and no new key was written" diff --git a/acceptance/bundle/config-remote-sync/split/target_variable/output.txt b/acceptance/bundle/config-remote-sync/split/target_variable/output.txt index 0fe0a2d21a8..ad659d5a682 100644 --- a/acceptance/bundle/config-remote-sync/split/target_variable/output.txt +++ b/acceptance/bundle/config-remote-sync/split/target_variable/output.txt @@ -4,6 +4,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Rename the notebook inside a target-block field that uses a variable + === Sync Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/split/target_variable/script b/acceptance/bundle/config-remote-sync/split/target_variable/script index da7ff428244..ed2138e1920 100644 --- a/acceptance/bundle/config-remote-sync/split/target_variable/script +++ b/acceptance/bundle/config-remote-sync/split/target_variable/script @@ -22,12 +22,10 @@ for task in r["tasks"]: PYEDIT title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync -t dev --save title "zzz keeps its own variable, and aaa is untouched" -echo trace diff.py databricks.yml.backup databricks.yml errcode trace grep -c 'var.dev_dir' databricks.yml errcode trace grep -c 'var.shared_dir' databricks.yml diff --git a/acceptance/bundle/config-remote-sync/split/variable_file_order/output.txt b/acceptance/bundle/config-remote-sync/split/variable_file_order/output.txt index 2850f34c624..aa50edc6777 100644 --- a/acceptance/bundle/config-remote-sync/split/variable_file_order/output.txt +++ b/acceptance/bundle/config-remote-sync/split/variable_file_order/output.txt @@ -4,6 +4,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Edit aaa, whose file position differs from its merged position + === Sync Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/split/variable_file_order/script b/acceptance/bundle/config-remote-sync/split/variable_file_order/script index b2e2b1d236f..f263d89c191 100644 --- a/acceptance/bundle/config-remote-sync/split/variable_file_order/script +++ b/acceptance/bundle/config-remote-sync/split/variable_file_order/script @@ -13,11 +13,9 @@ for task in r["tasks"]: PYEDIT title "Sync" -echo cp databricks.yml databricks.yml.backup errcode $CLI bundle config-remote-sync --save title "aaa keeps its own variable reference" -echo trace diff.py databricks.yml.backup databricks.yml errcode trace grep -c 'var.aaa_dir' databricks.yml rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/target_override/output.txt b/acceptance/bundle/config-remote-sync/target_override/output.txt index 4919594b609..7e9cec5a2d4 100644 --- a/acceptance/bundle/config-remote-sync/target_override/output.txt +++ b/acceptance/bundle/config-remote-sync/target_override/output.txt @@ -4,6 +4,7 @@ Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Modify fields in target override + === Detect and save changes Detected changes in 1 resource(s): diff --git a/acceptance/bundle/config-remote-sync/target_override/script b/acceptance/bundle/config-remote-sync/target_override/script index 671eb75ed23..85dddc9d19d 100755 --- a/acceptance/bundle/config-remote-sync/target_override/script +++ b/acceptance/bundle/config-remote-sync/target_override/script @@ -20,11 +20,9 @@ r["tags"]["owner"] = "data-team" EOF title "Detect and save changes" -echo cp databricks.yml databricks.yml.backup $CLI bundle config-remote-sync -t dev --save title "Configuration changes" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/task_rename_revert/script b/acceptance/bundle/config-remote-sync/task_rename_revert/script index 319a6731733..f5529ba0750 100644 --- a/acceptance/bundle/config-remote-sync/task_rename_revert/script +++ b/acceptance/bundle/config-remote-sync/task_rename_revert/script @@ -11,7 +11,6 @@ $CLI bundle deploy job_id="$(read_id.py "sample job's\"quotes")" title "Rename task to new_task_2 remotely" -echo edit_resource.py jobs $job_id <>> replay_tfstate.py terraform-requests.json terraform.tfstate === config-remote-sync migrates the terraform state in memory and detects the remote change + >>> [CLI] bundle config-remote-sync Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Detected changes in 1 resource(s): @@ -13,10 +15,12 @@ Resource: resources.jobs.test_job === The terraform state is untouched: config-remote-sync does not commit the migration + >>> find.py \.databricks/.*(\.tfstate|resources.*\.json) .databricks/bundle/default/terraform/terraform.tfstate === Saving writes the remote change into the config + >>> [CLI] bundle config-remote-sync --save Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Detected changes in 1 resource(s): @@ -41,6 +45,7 @@ resources: notebook_path: /Users/{{workspace_user_name}}/notebook === Cleanup + >>> [CLI] bundle destroy --auto-approve Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). The following resources will be deleted: diff --git a/acceptance/bundle/config-remote-sync/validation_errors/output.txt b/acceptance/bundle/config-remote-sync/validation_errors/output.txt index 4a956e718df..fde74f8a34e 100644 --- a/acceptance/bundle/config-remote-sync/validation_errors/output.txt +++ b/acceptance/bundle/config-remote-sync/validation_errors/output.txt @@ -5,7 +5,9 @@ Files: 3 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Set correct paths and add git info remotely + === Break local paths to simulate stale config + === Sync with broken local paths Detected changes in 2 resource(s): diff --git a/acceptance/bundle/config-remote-sync/validation_errors/script b/acceptance/bundle/config-remote-sync/validation_errors/script index 828ec206445..e2122145147 100644 --- a/acceptance/bundle/config-remote-sync/validation_errors/script +++ b/acceptance/bundle/config-remote-sync/validation_errors/script @@ -38,11 +38,9 @@ rm -rf pipeline_root src title "Sync with broken local paths" -echo cp databricks.yml databricks.yml.backup $CLI bundle config-remote-sync --save title "Configuration changes" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/variable_reference_parent/script b/acceptance/bundle/config-remote-sync/variable_reference_parent/script index c49921acfe5..336edbb1c01 100644 --- a/acceptance/bundle/config-remote-sync/variable_reference_parent/script +++ b/acceptance/bundle/config-remote-sync/variable_reference_parent/script @@ -13,7 +13,6 @@ $CLI bundle deploy job_id="$(read_id.py my_job)" title "Edit fields under complex variables" -echo edit_resource.py jobs $job_id <<'EOF' r["max_concurrent_runs"] = 5 @@ -29,17 +28,14 @@ del cluster["custom_tags"]["removed"] EOF title "Detect and save all changes" -echo cp databricks.yml databricks.yml.backup $CLI bundle config-remote-sync --save title "Configuration changes" -echo trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup title "Telemetry" -echo # The engine name is dropped because both EnvMatrix variants share this output. # skipped_changes_count includes changes under both immediate and distant variable ancestors. trace cat "$OUT_REQUESTS" | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_config_remote_sync_event | select(. != null) | del(.engine)' diff --git a/acceptance/bundle/debug/fetch-repository-info/script b/acceptance/bundle/debug/fetch-repository-info/script index 6f69b821a9b..abe6ff9e072 100644 --- a/acceptance/bundle/debug/fetch-repository-info/script +++ b/acceptance/bundle/debug/fetch-repository-info/script @@ -20,14 +20,14 @@ cleanup() { } trap cleanup EXIT -title "the root of a Git folder under /Repos\n" +title "the root of a Git folder under /Repos" trace $CLI bundle debug fetch-repository-info --path "$repo" -title "a subdirectory of it, which resolves to the same root\n" +title "a subdirectory of it, which resolves to the same root" trace $CLI bundle debug fetch-repository-info --path "$repo/subdir" -title "the root of a Git folder outside /Repos\n" +title "the root of a Git folder outside /Repos" trace $CLI bundle debug fetch-repository-info --path "$folder" -title "a path in no Git folder at all\n" +title "a path in no Git folder at all" trace $CLI bundle debug fetch-repository-info --path "/Workspace/Users/$CURRENT_USER_NAME" diff --git a/acceptance/bundle/deploy/empty-resource-id/create/output.txt b/acceptance/bundle/deploy/empty-resource-id/create/output.txt index b11534af573..8b76e7e9da5 100644 --- a/acceptance/bundle/deploy/empty-resource-id/create/output.txt +++ b/acceptance/bundle/deploy/empty-resource-id/create/output.txt @@ -1,5 +1,6 @@ === A successful create response without an ID fails the deployment + >>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Error: cannot create resources.volumes.volume1: create returned an empty resource id @@ -7,10 +8,12 @@ Error: cannot create resources.volumes.volume1: create returned an empty resourc Files: 0 uploaded, 0 deleted === The failed create does not save resource state + >>> print_state.py {} === The next plan still creates the resource + >>> [CLI] bundle plan create volumes.volume1 diff --git a/acceptance/bundle/deploy/empty-resource-id/update/output.txt b/acceptance/bundle/deploy/empty-resource-id/update/output.txt index 9650d45d203..ada20dd341a 100644 --- a/acceptance/bundle/deploy/empty-resource-id/update/output.txt +++ b/acceptance/bundle/deploy/empty-resource-id/update/output.txt @@ -6,6 +6,7 @@ Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === A successful rename response without an ID fails the deployment + >>> update_file.py databricks.yml myvolume mynewvolume >>> musterr [CLI] bundle deploy @@ -15,6 +16,7 @@ Error: cannot update_id resources.volumes.volume1: update returned an empty reso Files: 0 uploaded, 0 deleted === The failed rename preserves the original ID and state + >>> print_state.py { "resources.volumes.volume1": { @@ -29,6 +31,7 @@ Files: 0 uploaded, 0 deleted } === The next plan still updates the existing resource + >>> [CLI] bundle plan update volumes.volume1 diff --git a/acceptance/bundle/deploy/fail-on-active-runs/output.txt b/acceptance/bundle/deploy/fail-on-active-runs/output.txt index 91445d88273..92708ed504c 100644 --- a/acceptance/bundle/deploy/fail-on-active-runs/output.txt +++ b/acceptance/bundle/deploy/fail-on-active-runs/output.txt @@ -12,6 +12,7 @@ Error: job [MY_JOB_ID] is running Exit code: 1 === Expecting only 1 delete request to artifact_path/.internal folder from the first deploy + >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) [OUTPUT_DIR]/out.requests.txt { "method": "POST", diff --git a/acceptance/bundle/deploy/files/no-snapshot-sync/output.txt b/acceptance/bundle/deploy/files/no-snapshot-sync/output.txt index ab5fa74b709..39a499b943a 100644 --- a/acceptance/bundle/deploy/files/no-snapshot-sync/output.txt +++ b/acceptance/bundle/deploy/files/no-snapshot-sync/output.txt @@ -6,6 +6,7 @@ Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Check that test file is in workspace + >>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/test.py { "object_type": "FILE", @@ -19,6 +20,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Check that notebook is in workspace + >>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/notebook { "object_type": "NOTEBOOK", @@ -27,6 +29,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Check that deployment.json is synced correctly + >>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json { "object_type": "FILE", @@ -34,6 +37,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Remove .databricks directory to simulate a fresh deployment like in CI/CD environment + >>> rm -rf .databricks >>> rm test.py @@ -48,6 +52,7 @@ Files: 4 uploaded, 2 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged === Check that removed files are not in the workspace anymore + >>> errcode [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/test.py Error: Path (/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/test.py) doesn't exist. @@ -59,6 +64,7 @@ Error: Path (/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/notebook) d Exit code: 1 === Check the content of modified file + >>> [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/test_to_modify.py print('Modified!') diff --git a/acceptance/bundle/deploy/files/out-of-band-delete/output.txt b/acceptance/bundle/deploy/files/out-of-band-delete/output.txt index 4e5c2456633..4f15342d7bb 100644 --- a/acceptance/bundle/deploy/files/out-of-band-delete/output.txt +++ b/acceptance/bundle/deploy/files/out-of-band-delete/output.txt @@ -6,6 +6,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === After the initial deploy the python_file exists + >>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files/hello_world.py { "object_type": "FILE", @@ -13,9 +14,11 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Delete the remote bundle directory out-of-band (simulates deleting it in the UI) + >>> [CLI] workspace delete --recursive /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete === Redeploy reuses the stale snapshot, so the python_file is NOT re-uploaded + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files... Files: 0 uploaded, 0 deleted @@ -25,6 +28,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged Error: Path (/Workspace/Users/[USERNAME]/.bundle/out-of-band-delete/default/files/hello_world.py) doesn't exist. === Removing the sync snapshot forces a full re-upload, restoring the python_file + >>> rm -rf .databricks/bundle/default/sync-snapshots >>> [CLI] bundle deploy diff --git a/acceptance/bundle/deploy/force-lock-config/output.txt b/acceptance/bundle/deploy/force-lock-config/output.txt index 222e4c4dc6c..22dbfa6cd0d 100644 --- a/acceptance/bundle/deploy/force-lock-config/output.txt +++ b/acceptance/bundle/deploy/force-lock-config/output.txt @@ -1,8 +1,10 @@ === upload a stale lock file held by another user + >>> [CLI] workspace import /Workspace/Users/[USERNAME]/.bundle/test-bundle-force-lock/default/state/deploy.lock --format AUTO --file ./deploy.lock === bundle.deployment.lock.force is set in databricks.yml, so the stale lock is overridden + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-force-lock/default/files... Created jobs.foo diff --git a/acceptance/bundle/deploy/immutable/basic/output.txt b/acceptance/bundle/deploy/immutable/basic/output.txt index 6a92f410d0d..8beda578308 100644 --- a/acceptance/bundle/deploy/immutable/basic/output.txt +++ b/acceptance/bundle/deploy/immutable/basic/output.txt @@ -94,6 +94,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> title Check snapshot plan after deploy === Check snapshot plan after deploy + >>> [CLI] bundle plan -o json Building python_artifact... @@ -193,7 +194,8 @@ Updated jobs.my_job Files: 0 uploaded, 0 deleted Resources: 1 created, 1 changed, 1 deleted, 0 unchanged -=== Check that job uses a new snapshot: Snapshot changed as expected +=== Check that job uses a new snapshot: +Snapshot changed as expected >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/deploy/immutable/breakglass/output.txt b/acceptance/bundle/deploy/immutable/breakglass/output.txt index 1817666a6e4..debcff44f5a 100644 --- a/acceptance/bundle/deploy/immutable/breakglass/output.txt +++ b/acceptance/bundle/deploy/immutable/breakglass/output.txt @@ -1,5 +1,6 @@ === Deploy the immutable snapshot + >>> [CLI] bundle deploy Created internal_immutable_snapshots.immutable Created jobs.my_job @@ -8,16 +9,19 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged First snapshot path: /Workspace/Users/[UUID][0]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID][1]/snapshot/files/src/main.py === Break the glass out of band + >>> [CLI] api post /api/2.0/snapshots:breakGlass --json {"snapshot_content_path": "/Workspace/Users/[UUID][0]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID][1]/snapshot"} true === Deploy without --force fails + >>> musterr [CLI] bundle deploy Error: the last deployment was modified in the workspace. Changes might be overwritten by this deployment. To confirm it, use the --force flag Files: 0 uploaded, 0 deleted === Deploy with --force moves to a new snapshot + >>> [CLI] bundle deploy --force Recreated internal_immutable_snapshots.immutable Updated jobs.my_job @@ -27,6 +31,7 @@ Second snapshot path: /Workspace/Users/[UUID][0]/.snapshots/[SNAPSHOT_HASH]/[SNA OK: --force created a new snapshot path === Redeploy reuses the new snapshot + >>> [CLI] bundle deploy Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 2 unchanged @@ -34,18 +39,21 @@ Third snapshot path: /Workspace/Users/[UUID][0]/.snapshots/[SNAPSHOT_HASH]/[SNAP OK: new snapshot reused on redeploy === Save a plan for a content change, then break the glass before deploying it + >>> [CLI] bundle plan -o json >>> [CLI] api post /api/2.0/snapshots:breakGlass --json {"snapshot_content_path": "/Workspace/Users/[UUID][0]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID][2]/snapshot"} true === Deploying the saved plan without --force fails + >>> musterr [CLI] bundle deploy --plan plan.json Error: the last deployment was modified in the workspace. Changes might be overwritten by this deployment. To confirm it, use the --force flag Files: 0 uploaded, 0 deleted === Deploying the saved plan with --force moves to the planned snapshot + >>> [CLI] bundle deploy --plan plan.json --force Recreated internal_immutable_snapshots.immutable Updated jobs.my_job diff --git a/acceptance/bundle/deploy/immutable/can-manage/output.txt b/acceptance/bundle/deploy/immutable/can-manage/output.txt index cdcca39708d..933b665ce1a 100644 --- a/acceptance/bundle/deploy/immutable/can-manage/output.txt +++ b/acceptance/bundle/deploy/immutable/can-manage/output.txt @@ -1,5 +1,6 @@ === Deploy with a CAN_MANAGE permission entry + >>> [CLI] bundle deploy Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. @@ -20,6 +21,7 @@ Files: 0 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Check can_manage_principals sent to the snapshot create API + >>> print_requests.py //api/2.0/snapshots [ { diff --git a/acceptance/bundle/deploy/immutable/destroy-redeploy/output.txt b/acceptance/bundle/deploy/immutable/destroy-redeploy/output.txt index fb693d48163..4d9f47ea676 100644 --- a/acceptance/bundle/deploy/immutable/destroy-redeploy/output.txt +++ b/acceptance/bundle/deploy/immutable/destroy-redeploy/output.txt @@ -1,5 +1,6 @@ === Deploy the immutable snapshot + >>> [CLI] bundle deploy Created internal_immutable_snapshots.immutable Created jobs.my_job @@ -8,6 +9,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged First snapshot path: /Workspace/Users/[UUID][0]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID][1]/snapshot/files/src/main.py === Destroy the bundle + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.my_job @@ -17,6 +19,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted === Deploy again moves to a new snapshot instead of the one left behind + >>> [CLI] bundle deploy Created internal_immutable_snapshots.immutable Created jobs.my_job @@ -26,6 +29,7 @@ Second snapshot path: /Workspace/Users/[UUID][0]/.snapshots/[SNAPSHOT_HASH]/[SNA OK: deployment moved to a new snapshot === Redeploy reuses the new snapshot + >>> [CLI] bundle deploy Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 2 unchanged @@ -33,6 +37,7 @@ Third snapshot path: /Workspace/Users/[UUID][0]/.snapshots/[SNAPSHOT_HASH]/[SNAP OK: new snapshot reused on redeploy === Destroy and deploy again moves to yet another snapshot + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.my_job diff --git a/acceptance/bundle/deploy/immutable/legacy-state/output.txt b/acceptance/bundle/deploy/immutable/legacy-state/output.txt index 84dc036f7b8..104589f50d4 100644 --- a/acceptance/bundle/deploy/immutable/legacy-state/output.txt +++ b/acceptance/bundle/deploy/immutable/legacy-state/output.txt @@ -1,5 +1,6 @@ === Deploy an immutable snapshot + >>> [CLI] bundle deploy Created internal_immutable_snapshots.immutable Created jobs.my_job @@ -7,7 +8,9 @@ Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Rewrite the state the way a CLI without this feature wrote it + === Deploying on top of that state succeeds and moves to the current layout + >>> [CLI] bundle deploy Recreated internal_immutable_snapshots.immutable Updated jobs.my_job @@ -17,6 +20,7 @@ Snapshot path: /Workspace/Users/[UUID][0]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_H OK: deployed to the current snapshot layout === The check still applies to a deployment carried over from the older layout + >>> [CLI] api post /api/2.0/snapshots:breakGlass --json {"snapshot_content_path": "/Workspace/Users/[UUID][0]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID][1]/snapshot"} true diff --git a/acceptance/bundle/deploy/immutable/missing-snapshot/output.txt b/acceptance/bundle/deploy/immutable/missing-snapshot/output.txt index c938ad1143f..117318889d7 100644 --- a/acceptance/bundle/deploy/immutable/missing-snapshot/output.txt +++ b/acceptance/bundle/deploy/immutable/missing-snapshot/output.txt @@ -1,5 +1,6 @@ === Deploy an immutable snapshot + >>> [CLI] bundle deploy Created internal_immutable_snapshots.immutable Created jobs.my_job @@ -7,7 +8,9 @@ Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Point the state at a snapshot that is not there + === Deploying still works instead of failing on the snapshot that is not there + >>> [CLI] bundle deploy Recreated internal_immutable_snapshots.immutable Updated jobs.my_job diff --git a/acceptance/bundle/deploy/immutable/no-artifacts/output.txt b/acceptance/bundle/deploy/immutable/no-artifacts/output.txt index 687f952e97b..8c571528ae5 100644 --- a/acceptance/bundle/deploy/immutable/no-artifacts/output.txt +++ b/acceptance/bundle/deploy/immutable/no-artifacts/output.txt @@ -26,6 +26,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> title Check snapshot plan after deploy: === Check snapshot plan after deploy: + >>> [CLI] bundle plan -o json >>> jq .plan["resources.internal_immutable_snapshots.immutable"] plan.json diff --git a/acceptance/bundle/deploy/immutable/permissions-change/output.txt b/acceptance/bundle/deploy/immutable/permissions-change/output.txt index 7017c7b28f6..a7648db84ff 100644 --- a/acceptance/bundle/deploy/immutable/permissions-change/output.txt +++ b/acceptance/bundle/deploy/immutable/permissions-change/output.txt @@ -1,5 +1,6 @@ === Deploy without permissions + >>> [CLI] bundle deploy Created internal_immutable_snapshots.immutable Created jobs.my_job @@ -8,6 +9,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged First snapshot path: /Workspace/Users/[UUID][0]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID][1]/snapshot/files/src/main.py === Add permissions and redeploy + >>> [CLI] bundle deploy Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. diff --git a/acceptance/bundle/deploy/immutable/permissions-change/script b/acceptance/bundle/deploy/immutable/permissions-change/script index 458bfdd2c85..bc7ed87cbbb 100644 --- a/acceptance/bundle/deploy/immutable/permissions-change/script +++ b/acceptance/bundle/deploy/immutable/permissions-change/script @@ -26,7 +26,6 @@ SNAPSHOT_PATH_2=$($CLI jobs get "$JOB_ID" | jq -r '.settings.tasks[0].spark_pyth echo "Second snapshot path: $SNAPSHOT_PATH_2" title "Verify snapshot path changed after permissions change" -echo "" if [ "$SNAPSHOT_PATH_1" = "$SNAPSHOT_PATH_2" ]; then echo "ERROR: snapshot path did not change after permissions were added" exit 1 diff --git a/acceptance/bundle/deploy/immutable/select/output.txt b/acceptance/bundle/deploy/immutable/select/output.txt index 3b0e4543c60..758374dbc57 100644 --- a/acceptance/bundle/deploy/immutable/select/output.txt +++ b/acceptance/bundle/deploy/immutable/select/output.txt @@ -1,5 +1,6 @@ === Deploying only a job that does not use the snapshot records no snapshot in the metadata + >>> [CLI] bundle deploy --select jobs.other_job Created jobs.other_job Files: 0 uploaded, 0 deleted @@ -9,6 +10,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged, 2 not selected null === Deploy the immutable snapshot + >>> [CLI] bundle deploy Created internal_immutable_snapshots.immutable Created jobs.my_job @@ -18,6 +20,7 @@ Metadata snapshot path: /Workspace/Users/[UUID][0]/.snapshots/[SNAPSHOT_HASH]/[S OK: metadata points at the deployed snapshot === Deploying only that job after a content change keeps the deployed snapshot in the metadata + >>> [CLI] bundle deploy --select jobs.other_job Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged, 2 not selected @@ -25,6 +28,7 @@ Metadata snapshot path: /Workspace/Users/[UUID][0]/.snapshots/[SNAPSHOT_HASH]/[S OK: metadata still points at the deployed snapshot === A full deploy moves the metadata to the new snapshot + >>> [CLI] bundle deploy Recreated internal_immutable_snapshots.immutable Updated jobs.my_job diff --git a/acceptance/bundle/deploy/mlops-stacks/script b/acceptance/bundle/deploy/mlops-stacks/script index a1770334913..cdde2cce206 100644 --- a/acceptance/bundle/deploy/mlops-stacks/script +++ b/acceptance/bundle/deploy/mlops-stacks/script @@ -47,6 +47,6 @@ replace_ids.py trace $CLI bundle summary --var catalog_name=main -o json | jq -r '{experiment_id: .resources.experiments.experiment.id, model_id: .resources.registered_models.model.id, inference_job_id: .resources.jobs.batch_inference_job.id, training_job_id: .resources.jobs.model_training_job.id}' -title "Assert the batch inference job actually exists\n" +title "Assert the batch inference job actually exists" JOB_ID=$($CLI bundle summary --var catalog_name=main -o json | jq -r '.resources.jobs.batch_inference_job.id') $CLI jobs get "${JOB_ID}" | jq '{name: .settings.name}' diff --git a/acceptance/bundle/deploy/readplan/whl-upload/output.txt b/acceptance/bundle/deploy/readplan/whl-upload/output.txt index 04d64fe158a..fef495b6973 100644 --- a/acceptance/bundle/deploy/readplan/whl-upload/output.txt +++ b/acceptance/bundle/deploy/readplan/whl-upload/output.txt @@ -1,5 +1,6 @@ === bundle deploy uploads the wheel + >>> [CLI] bundle deploy Uploading .databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/whl-upload/default/files... @@ -25,6 +26,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted === bundle deploy --plan also uploads the wheel + >>> [CLI] bundle deploy --plan tmp.plan.json Uploading .databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/whl-upload/default/files... diff --git a/acceptance/bundle/deploy/resource-max-wait-zero/output.txt b/acceptance/bundle/deploy/resource-max-wait-zero/output.txt index 1f379a5692e..9cffd3182fd 100644 --- a/acceptance/bundle/deploy/resource-max-wait-zero/output.txt +++ b/acceptance/bundle/deploy/resource-max-wait-zero/output.txt @@ -1,5 +1,6 @@ === Zero cap: the create wait is skipped entirely + >>> DATABRICKS_BUNDLE_RESOURCE_MAX_WAIT=0 [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Warn: deploying resources.clusters.my_cluster: Not waiting for creation of resources.clusters.my_cluster (DATABRICKS_BUNDLE_RESOURCE_MAX_WAIT=0); it may still be in progress @@ -19,6 +20,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted === No cap: the deployment polls until the cluster is RUNNING + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created clusters.my_cluster diff --git a/acceptance/bundle/deploy/resource-max-wait/output.txt b/acceptance/bundle/deploy/resource-max-wait/output.txt index 9da7b2c014a..9d6d4cfad48 100644 --- a/acceptance/bundle/deploy/resource-max-wait/output.txt +++ b/acceptance/bundle/deploy/resource-max-wait/output.txt @@ -1,5 +1,6 @@ === A malformed value is rejected instead of falling back to the default wait + >>> DATABRICKS_BUNDLE_RESOURCE_MAX_WAIT=1m musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Error: invalid DATABRICKS_BUNDLE_RESOURCE_MAX_WAIT="1m": expected a non-negative number of seconds @@ -13,6 +14,7 @@ Error: invalid DATABRICKS_BUNDLE_RESOURCE_MAX_WAIT="-5": expected a non-negative Files: 0 uploaded, 0 deleted === A valid value is accepted + >>> DATABRICKS_BUNDLE_RESOURCE_MAX_WAIT=60 [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created jobs.my_job diff --git a/acceptance/bundle/deploy/wal/crash-after-create/output.txt b/acceptance/bundle/deploy/wal/crash-after-create/output.txt index 7c07a54e9ff..c48d31caccf 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/output.txt +++ b/acceptance/bundle/deploy/wal/crash-after-create/output.txt @@ -51,6 +51,7 @@ Exit code: [KILLED] } === Any other command recovers state + >>> assert_exists.py .databricks/bundle/default/resources.json >>> assert_not_exists.py .databricks/bundle/default/resources.json.wal diff --git a/acceptance/bundle/deploy/wal/failed-plan-no-wal/output.txt b/acceptance/bundle/deploy/wal/failed-plan-no-wal/output.txt index f1b616e23af..30bf3d671e7 100644 --- a/acceptance/bundle/deploy/wal/failed-plan-no-wal/output.txt +++ b/acceptance/bundle/deploy/wal/failed-plan-no-wal/output.txt @@ -1,5 +1,6 @@ === Deploy 1 (normal: creates the job and the committed state) + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created jobs.test_job @@ -7,6 +8,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Deploy 2 (planning fails, must not leave a WAL) + >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Error: cannot plan resources.jobs.test_job: reading id="[TEST_JOB_ID]": Fault injected by test. (403 INJECTED) @@ -25,6 +27,7 @@ Exit code: 1 >>> assert_not_exists.py .databricks/bundle/default/resources.json.wal === Deploy 3 (planning fails again, must not leave a WAL) + >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Error: cannot plan resources.jobs.test_job: reading id="[TEST_JOB_ID]": Fault injected by test. (403 INJECTED) @@ -43,6 +46,7 @@ Exit code: 1 >>> assert_not_exists.py .databricks/bundle/default/resources.json.wal === Deploy 4 (fault expired: recovers and succeeds) + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 0 uploaded, 0 deleted diff --git a/acceptance/bundle/deploy/wal/header-only-wal/output.txt b/acceptance/bundle/deploy/wal/header-only-wal/output.txt index 86ca2583663..b322d40610a 100644 --- a/acceptance/bundle/deploy/wal/header-only-wal/output.txt +++ b/acceptance/bundle/deploy/wal/header-only-wal/output.txt @@ -1,5 +1,6 @@ === First deploy (killed before recording the job, leaves a header-only WAL) + >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... [PROCESS_KILLED] @@ -10,6 +11,7 @@ Exit code: [KILLED] {"state_version":3,"cli_version":"[CLI_VERSION]","lineage":"[UUID]","serial":1} === Second deploy (killed again, leaves another header-only WAL) + >>> errcode [CLI] bundle deploy --force-lock Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... [PROCESS_KILLED] @@ -20,6 +22,7 @@ Exit code: [KILLED] {"state_version":3,"cli_version":"[CLI_VERSION]","lineage":"[UUID]","serial":1} === Third deploy (must recover and succeed, not blocked by the leftover WAL) + >>> errcode [CLI] bundle deploy --force-lock Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created jobs.test_job diff --git a/acceptance/bundle/deployment/bind/catalog/output.txt b/acceptance/bundle/deployment/bind/catalog/output.txt index b0fdbba006a..212707693b1 100644 --- a/acceptance/bundle/deployment/bind/catalog/output.txt +++ b/acceptance/bundle/deployment/bind/catalog/output.txt @@ -10,6 +10,7 @@ Successfully bound catalog with an id 'test-catalog-[UNIQUE_NAME]' Run 'bundle deploy' to deploy changes to your workspace === Deploy bundle + >>> [CLI] bundle deploy --force-lock --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated catalogs.foo @@ -23,15 +24,18 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Unbind catalog + >>> [CLI] bundle deployment unbind foo === Destroy bundle + >>> [CLI] bundle destroy --auto-approve All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default Destroy: 0 deleted === Read the pre-defined catalog again (expecting it still exists): + >>> [CLI] catalogs get test-catalog-[UNIQUE_NAME] { "comment": "This is a test catalog", @@ -39,5 +43,6 @@ Destroy: 0 deleted } === Test cleanup + >>> [CLI] catalogs delete test-catalog-[UNIQUE_NAME] --force 0 diff --git a/acceptance/bundle/deployment/bind/data_quality_monitor/output.txt b/acceptance/bundle/deployment/bind/data_quality_monitor/output.txt index eacd19e80e3..4925bf49830 100644 --- a/acceptance/bundle/deployment/bind/data_quality_monitor/output.txt +++ b/acceptance/bundle/deployment/bind/data_quality_monitor/output.txt @@ -15,6 +15,7 @@ Resources: Name: table,[UUID] === A malformed id is rejected + >>> musterr [CLI] bundle deployment bind monitor1 [UUID] --auto-approve Error: failed to fetch the resource, err: invalid data quality monitor id "[UUID]" (want ,) diff --git a/acceptance/bundle/deployment/bind/experiment/output.txt b/acceptance/bundle/deployment/bind/experiment/output.txt index 639a001fe61..220c198d226 100644 --- a/acceptance/bundle/deployment/bind/experiment/output.txt +++ b/acceptance/bundle/deployment/bind/experiment/output.txt @@ -28,4 +28,6 @@ Destroy: 0 deleted } === Test cleanup: -=== Delete the pre-defined experiment: 0 + +=== Delete the pre-defined experiment: +0 diff --git a/acceptance/bundle/deployment/bind/external_location/output.txt b/acceptance/bundle/deployment/bind/external_location/output.txt index 0fbc59805fd..266012cdacc 100644 --- a/acceptance/bundle/deployment/bind/external_location/output.txt +++ b/acceptance/bundle/deployment/bind/external_location/output.txt @@ -11,6 +11,7 @@ Successfully bound external_location with an id 'test-location-[UNIQUE_NAME]' Run 'bundle deploy' to deploy changes to your workspace === Deploy bundle + >>> [CLI] bundle deploy --force-lock --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated external_locations.test_location @@ -25,15 +26,18 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Unbind external location + >>> [CLI] bundle deployment unbind test_location === Destroy bundle + >>> [CLI] bundle destroy --auto-approve All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default Destroy: 0 deleted === Read the pre-defined external location again (expecting it still exists): + >>> [CLI] external-locations get test-location-[UNIQUE_NAME] { "name": "test-location-[UNIQUE_NAME]", @@ -42,5 +46,6 @@ Destroy: 0 deleted } === Test cleanup + >>> [CLI] external-locations delete test-location-[UNIQUE_NAME] 0 diff --git a/acceptance/bundle/deployment/bind/feature_spec/output.txt b/acceptance/bundle/deployment/bind/feature_spec/output.txt index ed574d28e2c..a983fb36577 100644 --- a/acceptance/bundle/deployment/bind/feature_spec/output.txt +++ b/acceptance/bundle/deployment/bind/feature_spec/output.txt @@ -7,6 +7,7 @@ >>> create_feature_spec_fixtures.py main.fspec_[UNIQUE_NAME] === Create the feature spec outside the bundle + >>> [CLI] api post /api/2.0/feature-engineering/feature-specs --json { "name": "main.fspec_[UNIQUE_NAME].boundspec", "features": [{"feature_lookup": {"table": "main.fspec_[UNIQUE_NAME].user_features", "lookup_keys": ["user_id"], "feature_names": ["age"]}}] @@ -32,6 +33,7 @@ Resources: URL: [DATABRICKS_URL]/explore/data/functions/main/fspec_[UNIQUE_NAME]/boundspec === The bound feature spec matches the config, so there is nothing to change + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/deployment/bind/job/generate-and-bind/output.txt b/acceptance/bundle/deployment/bind/job/generate-and-bind/output.txt index c80c5d84598..5a5949bda58 100644 --- a/acceptance/bundle/deployment/bind/job/generate-and-bind/output.txt +++ b/acceptance/bundle/deployment/bind/job/generate-and-bind/output.txt @@ -35,12 +35,14 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted === Check that job is bound and does not exist after bundle is destroyed: + >>> errcode [CLI] jobs get [JOB_ID] --output json Error: Job [JOB_ID] does not exist. Exit code: 1 === Delete the tmp folder: + >>> [CLI] workspace delete /Workspace/Users/[USERNAME]/python-[UNIQUE_NAME]/test >>> [CLI] workspace delete /Workspace/Users/[USERNAME]/python-[UNIQUE_NAME] diff --git a/acceptance/bundle/deployment/bind/job/generate-and-bind/script b/acceptance/bundle/deployment/bind/job/generate-and-bind/script index a049d45a9b8..e59a41d8869 100644 --- a/acceptance/bundle/deployment/bind/job/generate-and-bind/script +++ b/acceptance/bundle/deployment/bind/job/generate-and-bind/script @@ -1,4 +1,4 @@ -title "Create a pre-defined job:\n" +title "Create a pre-defined job:" PYTHON_NOTEBOOK_DIR="/Workspace/Users/${CURRENT_USER_NAME}/python-${UNIQUE_NAME}" PYTHON_NOTEBOOK="${PYTHON_NOTEBOOK_DIR}/test" diff --git a/acceptance/bundle/deployment/bind/job/job-abort-bind/output.txt b/acceptance/bundle/deployment/bind/job/job-abort-bind/output.txt index ee76570eda9..8e212f477d7 100644 --- a/acceptance/bundle/deployment/bind/job/job-abort-bind/output.txt +++ b/acceptance/bundle/deployment/bind/job/job-abort-bind/output.txt @@ -10,6 +10,7 @@ To proceed, use --auto-approve after reviewing the plan above. Exit code: 1 === Deploy bundle: + >>> [CLI] bundle deploy --force-lock Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-job-[UNIQUE_NAME]/files... Created jobs.foo @@ -17,6 +18,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Check that job is not bound and not updated with config from bundle: + >>> [CLI] jobs get [JOB_ID] { "job_id": [JOB_ID], @@ -33,4 +35,5 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } -=== Delete the pre-defined job [JOB_ID]:0 +=== Delete the pre-defined job [JOB_ID]: +0 diff --git a/acceptance/bundle/deployment/bind/job/job-abort-bind/script b/acceptance/bundle/deployment/bind/job/job-abort-bind/script index 57a9c5e57d8..63aedd7f285 100644 --- a/acceptance/bundle/deployment/bind/job/job-abort-bind/script +++ b/acceptance/bundle/deployment/bind/job/job-abort-bind/script @@ -1,6 +1,6 @@ cp -r $TESTDIR/../job-spark-python-task/hello_world.py . -title "Create a pre-defined job:\n" +title "Create a pre-defined job:" PYTHON_FILE="/Workspace/Users/${CURRENT_USER_NAME}/initial_hello_world.py" @@ -34,7 +34,7 @@ cleanup() { } trap cleanup EXIT -title "Expect binding to fail without an auto-approve flag:\n" +title "Expect binding to fail without an auto-approve flag:" trace errcode $CLI bundle deployment bind foo $JOB_ID &> out.bind-result.txt sed -n '/^Error:/,$p' out.bind-result.txt rm out.bind-result.txt diff --git a/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt b/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt index 994bd66bd38..87bef1e6521 100644 --- a/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt +++ b/acceptance/bundle/deployment/bind/job/job-spark-python-task/output.txt @@ -3,14 +3,17 @@ Created job with ID: [JOB_ID] === Bind job: + >>> [CLI] bundle deployment bind foo [JOB_ID] --auto-approve Successfully bound job with an id '[JOB_ID]' Run 'bundle deploy' to deploy changes to your workspace === Remove .databricks directory to simulate fresh deployment: + >>> rm -rf .databricks === Deploy bundle: + >>> [CLI] bundle deploy --force-lock --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-job-[UNIQUE_NAME]/files... Updated jobs.foo @@ -18,6 +21,7 @@ Files: 3 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Read the pre-defined job: + >>> [CLI] jobs get [JOB_ID] { "job_id": [JOB_ID], @@ -35,18 +39,22 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Unbind the job: + >>> [CLI] bundle deployment unbind foo === Remove .databricks directory to simulate fresh deployment: + >>> rm -rf .databricks === Destroy the bundle: + >>> [CLI] bundle destroy --auto-approve All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bind-job-[UNIQUE_NAME] Destroy: 0 deleted === Read the pre-defined job again (expecting it still exists): + >>> [CLI] jobs get [JOB_ID] { "job_id": [JOB_ID], @@ -63,4 +71,5 @@ Destroy: 0 deleted } } -=== Delete the pre-defined job [JOB_ID]:0 +=== Delete the pre-defined job [JOB_ID]: +0 diff --git a/acceptance/bundle/deployment/bind/job/job-spark-python-task/script b/acceptance/bundle/deployment/bind/job/job-spark-python-task/script index 8562dab5647..f0746595c55 100644 --- a/acceptance/bundle/deployment/bind/job/job-spark-python-task/script +++ b/acceptance/bundle/deployment/bind/job/job-spark-python-task/script @@ -1,4 +1,4 @@ -title "Create a pre-defined job:\n" +title "Create a pre-defined job:" PYTHON_FILE="/Workspace/Users/${CURRENT_USER_NAME}/initial_hello_world.py" diff --git a/acceptance/bundle/deployment/bind/job/stale-state/output.txt b/acceptance/bundle/deployment/bind/job/stale-state/output.txt index de95b53685f..c8bb35bf735 100644 --- a/acceptance/bundle/deployment/bind/job/stale-state/output.txt +++ b/acceptance/bundle/deployment/bind/job/stale-state/output.txt @@ -1,5 +1,6 @@ === Step 1: Deploy with job_1 only + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/stale_state_test/default/files... Created jobs.job_1 @@ -7,15 +8,19 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Step 2: Save stale local state (has only job_1, serial=1) + === Step 3: Add job_2 to config and deploy again + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/stale_state_test/default/files... Created jobs.job_2 Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 1 unchanged -=== Step 4: Record deployed job_2 ID from stateDeployed job_2 ID: [JOB_2_ID] +=== Step 4: Record deployed job_2 ID from state +Deployed job_2 ID: [JOB_2_ID] === Step 5: Restore stale local state (only job_1, serial=1) + === Step 6: Create external job and try to bind (should fail: remote state already has job_2) External job ID: [EXTERNAL_JOB_ID] diff --git a/acceptance/bundle/deployment/bind/job/stale-state/script b/acceptance/bundle/deployment/bind/job/stale-state/script index a931f94d52a..ba201a2a26d 100755 --- a/acceptance/bundle/deployment/bind/job/stale-state/script +++ b/acceptance/bundle/deployment/bind/job/stale-state/script @@ -16,7 +16,7 @@ echo "Deployed job_2 ID: $(read_id.py job_2)" title "Step 5: Restore stale local state (only job_1, serial=1)" cp stale_state.json "$state_file" -title "Step 6: Create external job and try to bind (should fail: remote state already has job_2)\n" +title "Step 6: Create external job and try to bind (should fail: remote state already has job_2)" job_id=$($CLI jobs create --json '{"name": "External Job"}' | jq -r '.job_id') add_repl "$job_id" EXTERNAL_JOB_ID echo "External job ID: $job_id" diff --git a/acceptance/bundle/deployment/bind/job/unreadable-state/output.txt b/acceptance/bundle/deployment/bind/job/unreadable-state/output.txt index fb685a8f8f8..0468505a228 100644 --- a/acceptance/bundle/deployment/bind/job/unreadable-state/output.txt +++ b/acceptance/bundle/deployment/bind/job/unreadable-state/output.txt @@ -1,5 +1,6 @@ === Deploy job_1 + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created jobs.job_1 @@ -7,6 +8,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Leave behind a WAL from a different lineage, as a crashed deploy would + === Bind must refuse: the state cannot be read, so we cannot tell if job_1 is already managed >>> errcode [CLI] bundle deployment bind job_1 [EXTERNAL_JOB_ID] --auto-approve @@ -16,6 +18,7 @@ Error: cannot check whether resources.jobs.job_1 is already bound: reading state Exit code: 1 === Nothing was bound: the WAL is untouched and no bind state was written + >>> diff.py resources.json.wal .databricks/bundle/default/resources.json.wal >>> assert_not_exists.py .databricks/bundle/default/resources.json.temp-bind diff --git a/acceptance/bundle/deployment/bind/job/unreadable-state/script b/acceptance/bundle/deployment/bind/job/unreadable-state/script index 8a3a2ed81d1..24141ec0061 100644 --- a/acceptance/bundle/deployment/bind/job/unreadable-state/script +++ b/acceptance/bundle/deployment/bind/job/unreadable-state/script @@ -4,7 +4,7 @@ trace $CLI bundle deploy title "Leave behind a WAL from a different lineage, as a crashed deploy would" cp resources.json.wal .databricks/bundle/default/ -title "Bind must refuse: the state cannot be read, so we cannot tell if job_1 is already managed\n" +title "Bind must refuse: the state cannot be read, so we cannot tell if job_1 is already managed" job_id=$($CLI jobs create --json '{"name": "External Job"}' | jq -r '.job_id') add_repl "$job_id" EXTERNAL_JOB_ID trace errcode $CLI bundle deployment bind job_1 "$job_id" --auto-approve diff --git a/acceptance/bundle/deployment/bind/postgres_data_api/output.txt b/acceptance/bundle/deployment/bind/postgres_data_api/output.txt index 6d40b177023..1b9249be911 100644 --- a/acceptance/bundle/deployment/bind/postgres_data_api/output.txt +++ b/acceptance/bundle/deployment/bind/postgres_data_api/output.txt @@ -1,6 +1,8 @@ === Enable the Data API on a database outside the bundle + === Deploy fails because the Data API is already enabled + >>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Error: cannot create resources.postgres_data_apis.data_api1: Data API is already enabled for this database (409 ALREADY_EXISTS) @@ -13,6 +15,7 @@ API message: Data API is already enabled for this database Files: 2 uploaded, 0 deleted === Bind adopts the existing Data API + >>> [CLI] bundle deployment bind data_api1 projects/test-project/branches/production/databases/test-database/data-api --auto-approve Successfully bound postgres_data_api with an id 'projects/test-project/branches/production/databases/test-database/data-api' Run 'bundle deploy' to deploy changes to your workspace @@ -54,6 +57,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Unbind leaves the Data API enabled + >>> [CLI] bundle deployment unbind data_api1 >>> [CLI] bundle summary @@ -73,6 +77,7 @@ Resources: } === Bind again; destroy turns off the Data API + >>> [CLI] bundle deployment bind data_api1 projects/test-project/branches/production/databases/test-database/data-api --auto-approve Successfully bound postgres_data_api with an id 'projects/test-project/branches/production/databases/test-database/data-api' Run 'bundle deploy' to deploy changes to your workspace diff --git a/acceptance/bundle/deployment/bind/registered-model-aliases/output.txt b/acceptance/bundle/deployment/bind/registered-model-aliases/output.txt index 8d8d43b39a2..1a16c0cc6e1 100644 --- a/acceptance/bundle/deployment/bind/registered-model-aliases/output.txt +++ b/acceptance/bundle/deployment/bind/registered-model-aliases/output.txt @@ -1,5 +1,6 @@ === A model created outside the bundle, with versions 1 and 2 registered the way mlflow.register_model does + >>> [CLI] registered-models create --json {"name": "test-registered-model-[UNIQUE_NAME]", "catalog_name": "main", "schema_name": "default"} { "full_name": "main.default.test-registered-model-[UNIQUE_NAME]" @@ -16,6 +17,7 @@ } === Bind reports the comment and aliases the deploy will set + >>> musterr [CLI] bundle deployment bind model1 main.default.test-registered-model-[UNIQUE_NAME] Plan: update resources.registered_models.model1 @@ -32,6 +34,7 @@ Successfully bound registered_model with an id 'main.default.test-registered-mod Run 'bundle deploy' to deploy changes to your workspace === Deploy sets them + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated registered_models.model1 @@ -80,6 +83,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Redeploy is a no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/deployment/bind/schema/output.txt b/acceptance/bundle/deployment/bind/schema/output.txt index 89ab1f23437..a7ec28c5106 100644 --- a/acceptance/bundle/deployment/bind/schema/output.txt +++ b/acceptance/bundle/deployment/bind/schema/output.txt @@ -1,35 +1,46 @@ === Bind schema test: + === Substitute variables in the template: -=== Create a pre-defined schema: { + +=== Create a pre-defined schema: +{ "full_name": "main.test-schema-[UUID]", "catalog_name": "main" } -=== Bind schema: Successfully bound schema with an id 'main.test-schema-[UUID]' +=== Bind schema: +Successfully bound schema with an id 'main.test-schema-[UUID]' Run 'bundle deploy' to deploy changes to your workspace -=== Deploy bundle: Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +=== Deploy bundle: +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated schemas.schema1 Files: 2 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged -=== Read the pre-defined schema: { +=== Read the pre-defined schema: +{ "full_name": "main.test-schema-[UUID]", "catalog_name": "main", "comment": "This schema was created from DABs" } === Unbind the schema: -=== Destroy the bundle: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +=== Destroy the bundle: +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default Destroy: 0 deleted -=== Read the pre-defined schema again (expecting it still exists): { +=== Read the pre-defined schema again (expecting it still exists): +{ "full_name": "main.test-schema-[UUID]", "catalog_name": "main", "comment": "This schema was created from DABs" } === Test cleanup: -=== Delete the pre-defined schema test-schema-[UUID]: 0 + +=== Delete the pre-defined schema test-schema-[UUID]: +0 diff --git a/acceptance/bundle/deployment/bind/volume/output.txt b/acceptance/bundle/deployment/bind/volume/output.txt index c71865859f3..89509ef00a8 100644 --- a/acceptance/bundle/deployment/bind/volume/output.txt +++ b/acceptance/bundle/deployment/bind/volume/output.txt @@ -6,6 +6,7 @@ } === Create a pre-defined volume: + >>> [CLI] bundle deployment bind volume1 main.test-schema-[UNIQUE_NAME].volume-[UNIQUE_NAME] --auto-approve Successfully bound volume with an id 'main.test-schema-[UNIQUE_NAME].volume-[UNIQUE_NAME]' Run 'bundle deploy' to deploy changes to your workspace diff --git a/acceptance/bundle/deployment/unbind/grants/output.txt b/acceptance/bundle/deployment/unbind/grants/output.txt index 56265eeccd3..ac869f48479 100644 --- a/acceptance/bundle/deployment/unbind/grants/output.txt +++ b/acceptance/bundle/deployment/unbind/grants/output.txt @@ -25,6 +25,7 @@ Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Grants should be the same as before unbind + >>> [CLI] grants get schema main.test-schema-[UNIQUE_NAME] --output json { "principal": "account users", diff --git a/acceptance/bundle/deployment/unbind/permissions/output.txt b/acceptance/bundle/deployment/unbind/permissions/output.txt index 4c7e4bde615..1107bb8a20a 100644 --- a/acceptance/bundle/deployment/unbind/permissions/output.txt +++ b/acceptance/bundle/deployment/unbind/permissions/output.txt @@ -27,6 +27,7 @@ Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Permissions should be the same as before unbind + >>> [CLI] jobs get-permissions [JOB_1_ID] --output json { "all_permissions": [ diff --git a/acceptance/bundle/destroy/all-resources/output.txt b/acceptance/bundle/destroy/all-resources/output.txt index 7949a6b7c08..027fbb7baa1 100644 --- a/acceptance/bundle/destroy/all-resources/output.txt +++ b/acceptance/bundle/destroy/all-resources/output.txt @@ -23,5 +23,6 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 2 deleted === Assert the bundle directory is deleted + >>> musterr [CLI] workspace get-status //Workspace/Users/[USERNAME]/.bundle/test-bundle Error: Path (//Workspace/Users/[USERNAME]/.bundle/test-bundle) doesn't exist. diff --git a/acceptance/bundle/destroy/force-lock-node-limit/output.txt b/acceptance/bundle/destroy/force-lock-node-limit/output.txt index 8226b36d367..552664c574f 100644 --- a/acceptance/bundle/destroy/force-lock-node-limit/output.txt +++ b/acceptance/bundle/destroy/force-lock-node-limit/output.txt @@ -1,5 +1,6 @@ === Deploy (creates the job and the deployment) + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created jobs.foo @@ -7,6 +8,7 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === destroy without --force-lock (lock write hits the node limit: aborts) + >>> errcode [CLI] bundle destroy --auto-approve Error: Failed to acquire deployment lock: access denied: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock Error: Failed to update, encountered possible permission error: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state @@ -20,6 +22,7 @@ Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html f Exit code: 1 === destroy with --force-lock (tolerates the node limit: proceeds and completes) + >>> [CLI] bundle destroy --force-lock --auto-approve Warn: Proceeding with destroy without a deployment lock: access denied: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock The following resources will be deleted: diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/output.txt b/acceptance/bundle/destroy/jobs-and-pipeline/output.txt index 68f1bc5e824..131fdc20a17 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/output.txt +++ b/acceptance/bundle/destroy/jobs-and-pipeline/output.txt @@ -10,9 +10,11 @@ Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Assert the snapshot file exists + >>> find.py --expect 1 ^.databricks/bundle/default/sync-snapshots/[a-z0-9]+\.json === Assert bundle deployment path is created + >>> [CLI] workspace get-status //Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] { "path": "/Users/[USERNAME]/.bundle/[UNIQUE_NAME]", @@ -20,6 +22,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Assert the pipeline is created + >>> [CLI] pipelines get [UUID] { "spec": { @@ -48,6 +51,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Destroy bundle + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.foo @@ -62,6 +66,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 2 deleted === Assert pipeline is deleted + >>> errcode [CLI] pipelines get [UUID] Error: The specified pipeline [UUID] was not found. @@ -82,6 +87,7 @@ Exit code: 1 .databricks/bundle/default/deployment.json === Assert bundle deployment path is deleted + >>> errcode [CLI] workspace get-status //Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Error: Path (//Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]) doesn't exist. diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/script b/acceptance/bundle/destroy/jobs-and-pipeline/script index 3ed9aaf3a01..b08e8ce05ef 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/script +++ b/acceptance/bundle/destroy/jobs-and-pipeline/script @@ -11,12 +11,12 @@ cleanup() { title "Assert pipeline is deleted" trace errcode $CLI pipelines get "${PIPELINE_ID}" - title "Assert job is deleted:\n" + title "Assert job is deleted:" errcode $CLI jobs get "${JOB_ID}" 2>&1 # Print the whole state tree so a lingering snapshot file or an empty scaffolding # dir (e.g. sync-snapshots/ or .internal/) both show up. - title "Assert destroy leaves the bundle state dir clean:\n" + title "Assert destroy leaves the bundle state dir clean:" trace find.py --dirs '^\.databricks(/|$)' title "Assert bundle deployment path is deleted" @@ -24,7 +24,7 @@ cleanup() { } trap cleanup EXIT -title "Assert the snapshot file does not exist:\n" +title "Assert the snapshot file does not exist:" if [[ ! -d ".databricks/bundle/default/sync-snapshots" ]]; then echo "Directory does not exist" else @@ -43,6 +43,6 @@ title "Assert the pipeline is created" PIPELINE_ID=$($CLI bundle summary -o json | jq -r '.resources.pipelines.bar.id') trace $CLI pipelines get "${PIPELINE_ID}" | nostamp | jq "{spec}" -title "Assert the job is created:\n" +title "Assert the job is created:" JOB_ID=$(read_id.py foo) $CLI jobs get "${JOB_ID}" | jq '{name: .settings.name}' diff --git a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script index 349bd172988..fa052dcb97f 100644 --- a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script +++ b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script @@ -8,7 +8,7 @@ # state is deleted), the lingering local lineage no longer matches the new one a fresh # redeploy mints, and every command that pulls remote state fails. Cover both cases. -title "Machine A: deploy, then destroy\n" +title "Machine A: deploy, then destroy" trace $CLI bundle deploy # Snapshot machine A's local state (lineage L1) while it exists, so the lingering-state @@ -19,17 +19,17 @@ cp -r .databricks .databricks.machineA trace $CLI bundle destroy --auto-approve -title "A clean destroy removes the local state file, so no stale lineage is left behind:\n" +title "A clean destroy removes the local state file, so no stale lineage is left behind:" trace find.py '^.databricks/bundle/default/(resources.json|terraform/terraform.tfstate)$' --expect 0 # Print the whole state tree so any empty scaffolding dir or stray file shows up. -title "and leaves the bundle state dir clean, with no empty scaffolding directories:\n" +title "and leaves the bundle state dir clean, with no empty scaffolding directories:" trace find.py --dirs '^\.databricks(/|$)' # Wipe whatever remains so the next deploy behaves like a fresh machine. rm -rf .databricks -title "Machine B: fresh deploy of the same bundle (mints a new lineage L2 remotely)\n" +title "Machine B: fresh deploy of the same bundle (mints a new lineage L2 remotely)" trace $CLI bundle deploy # Mask machine B's (remote) lineage. @@ -41,7 +41,7 @@ add_repl "$LINEAGE_B" LINEAGE_B rm -rf .databricks mv .databricks.machineA .databricks -title "Machine A with a lingering local state: any command that pulls remote state now fails\n" +title "Machine A with a lingering local state: any command that pulls remote state now fails" trace musterr $CLI bundle plan trace musterr $CLI bundle deploy trace musterr $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/destroy/root-path-name-target/output.txt b/acceptance/bundle/destroy/root-path-name-target/output.txt index ef3455ca0a2..723b379f84f 100644 --- a/acceptance/bundle/destroy/root-path-name-target/output.txt +++ b/acceptance/bundle/destroy/root-path-name-target/output.txt @@ -10,14 +10,17 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 0 deleted === Assert the bundle directory is deleted + >>> musterr [CLI] workspace get-status //Workspace/Users/[USERNAME]/.bundle/explicit-[UNIQUE_NAME]/test-bundle Error: Path (//Workspace/Users/[USERNAME]/.bundle/explicit-[UNIQUE_NAME]/test-bundle) doesn't exist. === Assert the prefix above it is kept + >>> [CLI] workspace get-status //Workspace/Users/[USERNAME]/.bundle/explicit-[UNIQUE_NAME] { "object_type": "DIRECTORY" } === Clean up + >>> [CLI] workspace delete //Workspace/Users/[USERNAME]/.bundle/explicit-[UNIQUE_NAME] --recursive diff --git a/acceptance/bundle/destroy/root-path-not-scoped/output.txt b/acceptance/bundle/destroy/root-path-not-scoped/output.txt index 1cc63137ca6..31f94b5ee61 100644 --- a/acceptance/bundle/destroy/root-path-not-scoped/output.txt +++ b/acceptance/bundle/destroy/root-path-not-scoped/output.txt @@ -10,10 +10,12 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 0 deleted === Assert the parent directory is kept + >>> [CLI] workspace get-status //Workspace/Users/[USERNAME]/.bundle/custom-root-[UNIQUE_NAME] { "object_type": "DIRECTORY" } === Clean up + >>> [CLI] workspace delete //Workspace/Users/[USERNAME]/.bundle/custom-root-[UNIQUE_NAME] --recursive diff --git a/acceptance/bundle/destroy/sibling-target/output.txt b/acceptance/bundle/destroy/sibling-target/output.txt index e4d4c286c87..cfe9bc029d1 100644 --- a/acceptance/bundle/destroy/sibling-target/output.txt +++ b/acceptance/bundle/destroy/sibling-target/output.txt @@ -10,23 +10,27 @@ Files: 2 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 0 unchanged === Destroy one target while the other is still deployed + >>> [CLI] bundle destroy -t dev --auto-approve All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev Destroy: 0 deleted === Assert the bundle directory is kept: the sibling target is still under it + >>> [CLI] workspace get-status //Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME] { "object_type": "DIRECTORY" } === Destroy the remaining target + >>> [CLI] bundle destroy -t prod --auto-approve All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/prod Destroy: 0 deleted === Assert the bundle directory is deleted + >>> musterr [CLI] workspace get-status //Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME] Error: Path (//Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]) doesn't exist. diff --git a/acceptance/bundle/dms/bind-unbind/output.txt b/acceptance/bundle/dms/bind-unbind/output.txt index d6d3fcd36e6..daaa7345a94 100644 --- a/acceptance/bundle/dms/bind-unbind/output.txt +++ b/acceptance/bundle/dms/bind-unbind/output.txt @@ -1,10 +1,12 @@ === Bind an existing job with deployment history enabled + >>> [CLI] bundle deployment bind foo [JOB_ID] --auto-approve Successfully bound job with an id '[JOB_ID]' Run 'bundle deploy' to deploy changes to your workspace === Deploy keeps the bound job ID and enables shadow history + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files... Updated jobs.foo @@ -20,6 +22,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Unbind preserves the shadow history marker and leaves the job intact + >>> [CLI] bundle deployment unbind foo >>> jq {features, state} .databricks/bundle/default/resources.json @@ -36,6 +39,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Rebind preserves the existing shadow history marker + >>> [CLI] bundle deployment bind foo [JOB_ID] --auto-approve Successfully bound job with an id '[JOB_ID]' Run 'bundle deploy' to deploy changes to your workspace @@ -49,6 +53,7 @@ Run 'bundle deploy' to deploy changes to your workspace } === A subsequent deploy continues managing the same job + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files... Files: 0 uploaded, 0 deleted diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index 02b3d5450f9..f8b66625dcc 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -1,5 +1,6 @@ === Deploy a schema, so the deployment and its first version exist + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default/files... Created schemas.foo @@ -60,6 +61,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === The plan is a no-op now and records the deployment_history feature; the service holds the schema at version 1 + >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -105,6 +107,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } === A destructive change without --auto-approve is declined: this console cannot prompt + >>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] >>> musterr [CLI] bundle deploy @@ -119,6 +122,7 @@ To proceed, use --auto-approve after reviewing the plan above. Files: 2 uploaded, 0 deleted === Nothing was recorded for the declined deploy - no version, so none to abort + >>> print_requests.py --dms --get //api/2.0/bundle { "method": "GET", @@ -126,6 +130,7 @@ Files: 2 uploaded, 0 deleted } === The recorded resources are unchanged by the decline: the schema is still at version 1 + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { @@ -147,6 +152,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted === After the destroy the deployment is gone, so the summary reports no history rather than a dangling id + >>> [CLI] bundle summary -o json false diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index e77719a1b69..fc3e2fe24c7 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -1,5 +1,6 @@ === Plan before first deploy: no deployment yet, so serial is still 0 + >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -16,6 +17,7 @@ create jobs.parent Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged === Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/files... Created jobs.child @@ -54,6 +56,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === The plan is a no-op now and records the deployment_history feature + >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -69,6 +72,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === The service lists both jobs, each recorded at version 1 + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { @@ -84,6 +88,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged ] === The raw recorded state the service holds for this deployment + >>> print_state.py --no-dms { "state_version": 3, diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index 94f0a68a122..6cae20bc707 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -1,5 +1,6 @@ === Deploy: the deployment is created carrying this bundle's metadata + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-metadata-before-[UNIQUE_NAME]/dev/files... Created schemas.foo @@ -60,6 +61,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === The plan records the deployment_history feature; the service holds the schema at version 1 + >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -84,6 +86,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ] === Rename the bundle and switch target+mode. The state path is pinned, so this is the same deployment, updated in place - the mask carries every field that changed + >>> update_file.py databricks.yml dms-metadata-before dms-metadata-after >>> [CLI] bundle deploy -t prod @@ -151,6 +154,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ] === Deploy again without touching anything: the record already says this, so no deployment update + >>> [CLI] bundle deploy -t prod Recommendation: target with 'mode: production' should set 'workspace.root_path' to make sure only one copy is deployed diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index c12e022ced2..32bda513d5f 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -1,5 +1,6 @@ === Deploy a schema with one grant: the grants node is recorded with its state + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-empty-grants-[UNIQUE_NAME]/default/files... Created schemas.foo @@ -79,6 +80,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Revoke the grant, so the grants node empties out + >>> update_file.py databricks.yml grants: [{principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA]}] grants: [] >>> [CLI] bundle deploy @@ -127,10 +129,12 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } === Plan again: reading state back from the service works and reports no work + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The plan is a no-op and records the deployment_history feature + >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -146,6 +150,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The grants node emptied out, so the service no longer lists it - only the schema remains + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { @@ -156,6 +161,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ] === The raw recorded state the service holds for this deployment + >>> print_state.py --no-dms { "state_version": 3, diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 74560decc83..1fd1d80ce28 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -1,5 +1,6 @@ === Enable history on existing remote file state without replacing the job + >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... Files: 3 uploaded, 0 deleted @@ -14,6 +15,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged } === Disable history without replacing the job + >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... Files: 0 uploaded, 0 deleted diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index a8b5781edc8..0e61b6c1ee1 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -1,5 +1,6 @@ === Deploy: the job is recorded + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/files... Created jobs.stuck @@ -60,6 +61,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === The plan is a no-op now and records the deployment_history feature + >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -75,6 +77,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === After deploy, the service lists the resource at version 1 + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { @@ -85,6 +88,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ] === The raw state after successful deploy + >>> print_state.py --no-dms { "state_version": 3, @@ -117,6 +121,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } === A resource that fails to delete is recorded as a failed operation carrying the error, so the history says why rather than showing the resource gone + >>> fault.py POST /api/2.2/jobs/delete 400 0 1 INVALID_PARAMETER_VALUE >>> musterr [CLI] bundle destroy --auto-approve @@ -175,6 +180,7 @@ API message: Fault injected by test. } === After failure, the service shows the operation failed + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { @@ -185,6 +191,7 @@ API message: Fault injected by test. ] === The failed operation records the error message + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/2 -o json [ { @@ -196,6 +203,7 @@ API message: Fault injected by test. ] === The local state file keeps the resource and its serial after the failed delete + >>> print_state.py --no-dms { "state_version": 3, @@ -228,6 +236,7 @@ API message: Fault injected by test. } === The remote checkpoint stays at serial 1 while DMS advances to version 2 + >>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/resources.json --format RAW {"serial":1} diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 2e599afbde7..88890f0e2a9 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -1,5 +1,6 @@ === Deploy the schema, so there is a resource to recreate + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate-[UNIQUE_NAME]/default/files... Created schemas.foo @@ -60,6 +61,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === The plan is a no-op and records the deployment_history feature + >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -75,6 +77,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === After deploy, the service lists the resource at version 1 + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { @@ -85,6 +88,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ] === The raw state after successful deploy + >>> print_state.py --no-dms { "state_version": 3, @@ -107,6 +111,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } === A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy + >>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] >>> fault.py POST /api/2.1/unity-catalog/schemas 400 0 1 INVALID_PARAMETER_VALUE @@ -177,10 +182,12 @@ Files: 2 uploaded, 0 deleted } === The resource is not listed: state is what projects a resource, and the failed recreate left none + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [] === The raw state after failed recreate shows no resource + >>> print_state.py --no-dms { "state_version": 3, @@ -193,4 +200,4 @@ Files: 2 uploaded, 0 deleted "state": {} } -=== Planning after pulling remote file state creates the schema, which is what has to happen: it no longer exists \ No newline at end of file +=== Planning after pulling remote file state creates the schema, which is what has to happen: it no longer exists diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index e3b75a673de..0f89039c104 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -1,5 +1,6 @@ === Deploy the job and its permissions + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/files... Created jobs.foo @@ -8,6 +9,7 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === The plan records the deployment_history feature and the job + >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -23,6 +25,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === The service lists the job at version 1 + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { @@ -38,6 +41,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged ] === The raw recorded state after deploy + >>> print_state.py --no-dms { "state_version": 3, @@ -92,6 +96,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged } === A permissions update that fails records the failure with the permissions it still has. The resource ID here is a path rather than a number, and the state is the embedded permission list, so this covers a shape the resource tests do not + >>> update_file.py databricks.yml CAN_VIEW CAN_MANAGE >>> fault.py PUT /api/2.0/permissions/jobs/* 403 0 1 PERMISSION_DENIED @@ -108,6 +113,7 @@ API message: Fault injected by test. Files: 1 uploaded, 0 deleted === The service lists the job with the failure + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { @@ -123,6 +129,7 @@ Files: 1 uploaded, 0 deleted ] === The failed operation (version 2) records the error message + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[DEPLOYMENT_ID]/versions/2 -o json [ { @@ -134,6 +141,7 @@ Files: 1 uploaded, 0 deleted ] === The raw recorded state after the failure + >>> print_state.py --no-dms { "state_version": 3, diff --git a/acceptance/bundle/dms/immutable-folder/output.txt b/acceptance/bundle/dms/immutable-folder/output.txt index 1e3d4e5c0e6..8173a6fb530 100644 --- a/acceptance/bundle/dms/immutable-folder/output.txt +++ b/acceptance/bundle/dms/immutable-folder/output.txt @@ -1,5 +1,6 @@ === Deploying with an immutable folder records the resolved snapshot path as the deployment's file_path + >>> [CLI] bundle deploy Created internal_immutable_snapshots.immutable Created jobs.foo @@ -15,6 +16,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === A new snapshot moves file_path, which is then updated on the deployment + >>> [CLI] bundle deploy Recreated internal_immutable_snapshots.immutable Updated jobs.foo @@ -30,6 +32,7 @@ Resources: 1 created, 1 changed, 1 deleted, 0 unchanged } === An unchanged snapshot keeps the recorded file_path + >>> [CLI] bundle deploy Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 2 unchanged diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index cdb857ee2ba..80487e6f218 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -1,5 +1,6 @@ === Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; other bundle/dms tests cover it. + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/files... Created jobs.five @@ -11,6 +12,7 @@ Files: 2 uploaded, 0 deleted Resources: 5 created, 0 changed, 0 deleted, 0 unchanged === History telemetry survives version completion and counts all five staged operations + >>> cat [OUTPUT_DIR]/out.requests.txt { "staged_operation_count": 5, @@ -31,6 +33,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.two", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[TWO_ID]", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} === The plan records the deployment_history feature + >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -46,6 +49,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged === The service lists all five resources at version 1 + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { @@ -76,6 +80,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged ] === The raw recorded state the service holds for this deployment + >>> print_state.py --no-dms { "state_version": 3, @@ -180,18 +185,21 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged } === Redeploy with no changes: nothing is applied, so no version is created at all - the deployment's version tracks state changes, like serial + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/files... Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 5 unchanged === A deploy with no version omits operation buffer telemetry + >>> cat [OUTPUT_DIR]/out.requests.txt false >>> print_requests.py --dms //api/2.0/bundle --sort --oneline === The plan records the deployment_history feature + >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -207,6 +215,7 @@ false Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged === The service still lists all five resources, now at version 2 + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { @@ -237,6 +246,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged ] === The raw recorded state after redeploy + >>> print_state.py --no-dms { "state_version": 3, diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 14f650c01dd..8e0179f35a3 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -1,5 +1,6 @@ === Deploy job foo and pipeline bar; both are recorded at version 1 + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files... Created jobs.foo @@ -28,6 +29,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === The plan is a no-op now and records the deployment_history feature + >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -43,6 +45,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === The raw recorded state the service holds for this deployment + >>> print_state.py --no-dms { "state_version": 3, @@ -99,10 +102,12 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged } === Plan without touching anything: the deployment stamp must not show as drift + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === Update only the pipeline (not the job), then redeploy: only the pipeline's recorded version bumps to 2 + >>> update_file.py databricks.yml serverless: true serverless: true development: true @@ -213,6 +218,7 @@ Warning: The "development" property of pipelines is deprecated. Use "mode: devel } === The plan shows the updated pipeline at serial 2 with the deployment_history feature + >>> [CLI] bundle plan -o json Warning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes. at resources.pipelines.bar.development @@ -236,6 +242,7 @@ Warning: The "development" property of pipelines is deprecated. Use "mode: devel Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === The raw recorded state the service holds after the update + >>> print_state.py --no-dms { "state_version": 3, @@ -293,6 +300,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged } === Plan again: no residual drift, and the version stamps are ignored + >>> [CLI] bundle plan Warning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes. at resources.pipelines.bar.development diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 8380cc3885c..06a1730eadc 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -1,5 +1,6 @@ === First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/files... Files: 2 uploaded, 0 deleted @@ -30,6 +31,7 @@ deployment_history_enabled true } === The plan records the deployment_history feature + >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -45,10 +47,12 @@ deployment_history_enabled true Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged === The service lists no resources for the deployment + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [] === The raw recorded state the service holds for this deployment + >>> print_state.py --no-dms { "state_version": 3, @@ -62,6 +66,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged } === Redeploy: the deployment is resolved from that node, so no second deployment is created + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/files... Files: 0 uploaded, 0 deleted @@ -86,6 +91,7 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged } === The plan records the deployment_history feature + >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -101,10 +107,12 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged === The service still lists no resources + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [] === The raw recorded state after redeploy + >>> print_state.py --no-dms { "state_version": 3, diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index c8079249d34..6f5e558dbc9 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -1,5 +1,6 @@ === Deploy so there is a recorded resource to recreate + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-upload-fails-once-[UNIQUE_NAME]/default/files... Created schemas.foo @@ -7,6 +8,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === The plan records the deployment_history feature and the schema + >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -22,6 +24,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The service lists the schema at version 1 + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { @@ -32,6 +35,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ] === The raw recorded state after deploy + >>> print_state.py --no-dms { "state_version": 3, @@ -54,6 +58,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } === A recreate records twice, and the first one fails: the second must still carry the sequence id staging left, because a rejected update earned no new one + >>> fault.py PATCH /api/2.0/bundle/** 500 0 1 >>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] @@ -69,6 +74,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 0 unchanged === A failed history upload is counted separately from the queued operations + >>> cat [OUTPUT_DIR]/out.requests.txt { "staged_operation_count": 1, @@ -81,6 +87,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged } === The service lists the schema with the failure + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { @@ -91,6 +98,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged ] === The raw recorded state after the failure + >>> print_state.py --no-dms { "state_version": 3, @@ -166,6 +174,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged } === A completion failure does not block destroy or remote file cleanup + >>> fault.py POST /api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/complete 400 0 1 >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/dms/plan-without-dms/output.txt b/acceptance/bundle/dms/plan-without-dms/output.txt index db3e622639d..1e6c6cf3dd9 100644 --- a/acceptance/bundle/dms/plan-without-dms/output.txt +++ b/acceptance/bundle/dms/plan-without-dms/output.txt @@ -1,11 +1,13 @@ === Generate and save plan WITHOUT DMS recording + >>> jq {features} { "features": null } === Enable DMS recording and try to deploy the non-DMS plan + >>> musterr env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy --plan tmp.plan.json Error: this plan was created for a different set of state features than the target now has; run 'bundle plan' again diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index 3f00aae0d81..3c02e87950c 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -1,5 +1,6 @@ === Deploying from a git repo records where the source came from: the version carries git_info, workspace_info and the target's deployment_mode + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/files... Created jobs.foo @@ -52,6 +53,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === The plan is a no-op and records the deployment_history feature + >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -67,6 +69,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version + >>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[DEPLOYMENT_ID] { "target_name": "dev", @@ -83,6 +86,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } === The service lists the deployed job + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { @@ -93,6 +97,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ] === The raw recorded state the service holds for this deployment + >>> print_state.py --no-dms -t dev { "state_version": 3, diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 4af2dd20f6d..23129df751d 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -1,5 +1,6 @@ === A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource + >>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/files... Error: cannot create resources.jobs.doomed: Node type nonexistent-node-type is not supported. Supported node types: [NODE_TYPES] (400 INVALID_PARAMETER_VALUE) @@ -79,6 +80,7 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged } === The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed + >>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources {} @@ -98,6 +100,7 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged } === Redeploying after pulling remote file state still creates the failed resource: it was never applied, so it must not be skipped as unchanged + >>> [CLI] bundle plan create jobs.doomed diff --git a/acceptance/bundle/dms/stale-local-state/output.txt b/acceptance/bundle/dms/stale-local-state/output.txt index b1f26591d10..c9e64c60af2 100644 --- a/acceptance/bundle/dms/stale-local-state/output.txt +++ b/acceptance/bundle/dms/stale-local-state/output.txt @@ -1,5 +1,6 @@ === Setup (LOG.setup): reconstruct full version 1 without a history marker, then deploy version 2 + >>> jq -c {serial, features} .databricks/resources.v1.json {"serial":1,"features":null} @@ -7,12 +8,14 @@ {"serial":2,"features":{"deployment_history_shadow":{}}} === File serial 3 ahead of DMS version 2 is allowed + >>> update_file.py .databricks/bundle/default/resources.json "serial": 2 "serial": 3 >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === 1. Config false and env false: newer remote file state replaces the local copy + >>> cp .databricks/resources.v1.json .databricks/bundle/default/resources.json >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle summary --force-pull -o json @@ -23,6 +26,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_requests.py --dms //resources --get --oneline === 2. Config removed and env false: restore the same snapshot, then read the remote file state + >>> cp .databricks/resources.v1.json .databricks/bundle/default/resources.json >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle summary --force-pull -o json @@ -33,6 +37,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_requests.py --dms //resources --get --oneline === Destroy follows newer local state even when configuration enables history + >>> env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.foo diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index ebc25578f19..faf536c72ac 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -1,5 +1,6 @@ === Save a plan, then deploy it: records version 1 + >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -28,6 +29,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === The raw recorded state the service holds for this deployment + >>> print_state.py --no-dms { "state_version": 3, @@ -60,17 +62,20 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === A saved plan carries the version it was built against + >>> [CLI] bundle plan -o json { "serial": 1 } === Replay: re-deploying the same plan is rejected - it predates the recorded version + >>> musterr [CLI] bundle deploy --plan tmp.plan.json Error: plan lineage "" does not match state lineage "[UUID]"; the state may have been modified by another process === Only the first deploy recorded a version; the rejected replay created none + >>> print_requests.py --dms //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-stale-plan-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default"}}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.foo"}], "version_type": "VERSION_TYPE_DEPLOY"}} @@ -78,6 +83,7 @@ Error: plan lineage "" does not match state lineage "[UUID]"; the state may have {"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} === Save a plan against the recorded deployment, then move the deployment on + >>> jq {serial} tmp.plan2.json { "serial": 1 @@ -97,6 +103,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Replaying the older plan is rejected: same deployment, outdated version + >>> musterr [CLI] bundle deploy --plan tmp.plan2.json Error: plan serial 1 does not match state serial 2; the state has been modified since the plan was created. Please run 'bundle plan' again diff --git a/acceptance/bundle/dms/state-features/output.txt b/acceptance/bundle/dms/state-features/output.txt index 576a9c9f6c1..8328665355b 100644 --- a/acceptance/bundle/dms/state-features/output.txt +++ b/acceptance/bundle/dms/state-features/output.txt @@ -1,5 +1,6 @@ === Deploy keeps resource state in the file and enforces best-effort history + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-state-features-[UNIQUE_NAME]/default/files... Created jobs.foo @@ -35,12 +36,14 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Reopening reads resources from the file and produces a no-op plan + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_requests.py --dms //resources --get --oneline === Disabling deployment history allows plan and deploy, and removes the file marker + >>> env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle plan update jobs.foo @@ -58,10 +61,12 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> print_requests.py --dms //api/2.0/bundle --oneline === Removing the YAML setting also leaves history disabled + >>> env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Destroy also uses the disabled setting + >>> env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.foo diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index 41c9d29e8e2..3a414ec1815 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -1,5 +1,6 @@ === Deploy: the state write records the resource + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-successful-recreate-[UNIQUE_NAME]/default/files... Created schemas.foo @@ -60,6 +61,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === The plan is a no-op now and records the deployment_history feature + >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -75,6 +77,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The service lists the schema, recorded at version 1 + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { @@ -85,6 +88,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ] === The raw recorded state the service holds for this deployment + >>> print_state.py --no-dms { "state_version": 3, @@ -107,6 +111,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } === Recreate writes state twice - the entry is dropped, then the new resource is saved + >>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] >>> [CLI] bundle deploy --auto-approve @@ -172,6 +177,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged } === The recreate is done: nothing left to plan, and the serial moved to 2 + >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], @@ -187,6 +193,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The service lists the schema at version 2 with recreate action + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [ { @@ -197,6 +204,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ] === The raw recorded state after recreate + >>> print_state.py --no-dms { "state_version": 3, @@ -219,6 +227,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } === Destroy: the delete is recorded with the id and no state + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.schemas.foo diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index e4feb0d5398..7a555352349 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -1,5 +1,6 @@ === Version creation fails, but file-backed deployment still succeeds + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... Warn: Could not enable deployment history for this deploy: failed to create deployment version: Internal error @@ -36,6 +37,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged [] === The next resource update also succeeds while history version creation fails + >>> update_file.py databricks.yml name: foo name: renamed >>> [CLI] bundle deploy diff --git a/acceptance/bundle/escaped_refs/output.txt b/acceptance/bundle/escaped_refs/output.txt index 6dec4fd2c6b..fd670f19198 100644 --- a/acceptance/bundle/escaped_refs/output.txt +++ b/acceptance/bundle/escaped_refs/output.txt @@ -1,5 +1,6 @@ === Validate: escapes are still escaped in the config + >>> [CLI] bundle validate -o json { "generic_config": "{\"source\": {\"path\": \"landing/$${p_folder}/$${source_file}_*.csv\"}, \"schema\": {\"columns\": [{\"name\": \"w_batch_id\", \"expression\": \"$${batch_id}\"}, {\"name\": \"w_business_dt\", \"expression\": \"cast('$${business_date[0..9]}' as date)\"}]}, \"target\": {\"table\": \"main.$${p_env_stg}.$${target_table}\"}}", @@ -7,6 +8,7 @@ } === Plan (json) + >>> [CLI] bundle plan -o json === Deploy: payload sent to the jobs API diff --git a/acceptance/bundle/escaped_refs/script b/acceptance/bundle/escaped_refs/script index 2cce0c25096..6b8430ee926 100644 --- a/acceptance/bundle/escaped_refs/script +++ b/acceptance/bundle/escaped_refs/script @@ -22,7 +22,7 @@ nostamp < .databricks/tmp.plan.json > out.plan.json # Not traced: the READPLAN variant adds "--plan ...", and the point of that variant is # that the payload below is identical either way. -title "Deploy: payload sent to the jobs API\n" +title "Deploy: payload sent to the jobs API" $CLI bundle deploy $(readplanarg .databricks/tmp.plan.json) # --nostamp: under deployment-history recording the job payload carries the DMS deployment # stamp; drop it so the assertion is the same whether or not recording is on. diff --git a/acceptance/bundle/experimental/skip_name_prefix_for_schema/output.txt b/acceptance/bundle/experimental/skip_name_prefix_for_schema/output.txt index 7812e510ea7..7041f232388 100644 --- a/acceptance/bundle/experimental/skip_name_prefix_for_schema/output.txt +++ b/acceptance/bundle/experimental/skip_name_prefix_for_schema/output.txt @@ -1,5 +1,6 @@ === experimental.skip_name_prefix_for_schema is false + >>> [CLI] bundle validate -o json "dev_[USERNAME]_my_schema" @@ -10,6 +11,7 @@ "my_schema" === experimental.skip_name_prefix_for_schema is true + >>> [CLI] bundle validate -o json -t one "my_schema" diff --git a/acceptance/bundle/generate/auto-bind/output.txt b/acceptance/bundle/generate/auto-bind/output.txt index e7a43c9c50e..79602f58fbf 100644 --- a/acceptance/bundle/generate/auto-bind/output.txt +++ b/acceptance/bundle/generate/auto-bind/output.txt @@ -7,6 +7,7 @@ Created job with ID: [JOB_ID] >>> [CLI] workspace import /Workspace/Users/[USERNAME]/python-[UNIQUE_NAME]/test --file test.py --language PYTHON === Generate and bind in one step: + >>> [CLI] bundle generate job --key test_job --existing-job-id [JOB_ID] --config-dir resources --source-dir src --bind File successfully saved to src/test.py Job configuration successfully saved to resources/test_job.job.yml @@ -19,6 +20,7 @@ test.py name: auto-bind-job-[UNIQUE_NAME] === Deploy the bound job: + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/auto-bind-test-[UNIQUE_NAME]/files... Updated jobs.test_job @@ -26,6 +28,7 @@ Files: 5 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Destroy the bundle: + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.test_job @@ -35,12 +38,14 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted === Check that job is bound and does not exist after bundle is destroyed: + >>> errcode [CLI] jobs get [JOB_ID] --output json Error: Job [JOB_ID] does not exist. Exit code: 1 === Delete the tmp folder: + >>> [CLI] workspace delete /Workspace/Users/[USERNAME]/python-[UNIQUE_NAME]/test >>> [CLI] workspace delete /Workspace/Users/[USERNAME]/python-[UNIQUE_NAME] diff --git a/acceptance/bundle/generate/auto-bind/script b/acceptance/bundle/generate/auto-bind/script index 542dc0b96d6..533259e8019 100755 --- a/acceptance/bundle/generate/auto-bind/script +++ b/acceptance/bundle/generate/auto-bind/script @@ -1,4 +1,4 @@ -title "Create a pre-defined job:\n" +title "Create a pre-defined job:" PYTHON_NOTEBOOK_DIR="/Workspace/Users/${CURRENT_USER_NAME}/python-${UNIQUE_NAME}" PYTHON_NOTEBOOK="${PYTHON_NOTEBOOK_DIR}/test" diff --git a/acceptance/bundle/generate/dashboard-inplace/output.txt b/acceptance/bundle/generate/dashboard-inplace/output.txt index 76f6dc6195b..b67e6df7a3e 100644 --- a/acceptance/bundle/generate/dashboard-inplace/output.txt +++ b/acceptance/bundle/generate/dashboard-inplace/output.txt @@ -3,6 +3,7 @@ {} === deploy initial dashboard + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dashboard update inplace/default/files... Created dashboards.test_dashboard @@ -10,6 +11,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === update the dashboard + >>> [CLI] lakeview update [DASHBOARD_ID] --serialized-dashboard {"a":"b"} { "create_time": "[TIMESTAMP]", @@ -27,6 +29,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> fault.py GET /api/2.0/bundle/deployments/* 403 0 1 PERMISSION_DENIED === update the dashboard file using bundle generate + >>> [CLI] bundle generate dashboard --resource test_dashboard --force Writing dashboard to dash.lvdash.json diff --git a/acceptance/bundle/generate/genie_space_inplace/output.txt b/acceptance/bundle/generate/genie_space_inplace/output.txt index 45b7eb0ab89..3861e18afc4 100644 --- a/acceptance/bundle/generate/genie_space_inplace/output.txt +++ b/acceptance/bundle/generate/genie_space_inplace/output.txt @@ -3,6 +3,7 @@ {} === deploy initial genie space + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/genie space update inplace/default/files... Created genie_spaces.test_genie_space @@ -10,6 +11,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === update the genie space + >>> [CLI] genie update-space [GENIE_SPACE_ID] --serialized-space {"a":"b"} { "etag": "2", @@ -23,6 +25,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> fault.py GET /api/2.0/bundle/deployments/* 403 0 1 PERMISSION_DENIED === update the genie space file using bundle generate + >>> [CLI] bundle generate genie-space --resource test_genie_space --force Writing genie space to space.geniespace.json diff --git a/acceptance/bundle/generate/pipeline_and_deploy/output.txt b/acceptance/bundle/generate/pipeline_and_deploy/output.txt index c63cfee74d5..01b353547e9 100644 --- a/acceptance/bundle/generate/pipeline_and_deploy/output.txt +++ b/acceptance/bundle/generate/pipeline_and_deploy/output.txt @@ -1,12 +1,15 @@ === Upload files to workspace + >>> [CLI] workspace import /Workspace/Users/[USERNAME]/notebook-[UNIQUE_NAME].py --file notebook.py --format AUTO --overwrite >>> [CLI] workspace import /Workspace/Users/[USERNAME]/test-[UNIQUE_NAME].py --file test.py --format AUTO --overwrite -=== Create a pipeline that references the filesCreated pipeline +=== Create a pipeline that references the files +Created pipeline -=== Generate bundle config from the pipelineFile successfully saved to src/notebook-[UNIQUE_NAME].py +=== Generate bundle config from the pipeline +File successfully saved to src/notebook-[UNIQUE_NAME].py File successfully saved to src/test-[UNIQUE_NAME].py Pipeline configuration successfully saved to resources/out.pipeline.yml Warning: Generated configuration is not included in the bundle @@ -19,19 +22,23 @@ include: === Verify generated yaml has expected fields + === Deploy the generated bundle + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline_and_deploy-[UNIQUE_NAME]/default/files... Files: 7 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 0 unchanged === Destroy the deployed bundle + >>> [CLI] bundle destroy --auto-approve All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/pipeline_and_deploy-[UNIQUE_NAME]/default Destroy: 0 deleted === Cleanup: delete the original pipeline and files + >>> errcode [CLI] pipelines delete [PIPELINE_ID] >>> errcode [CLI] workspace delete /Workspace/Users/[USERNAME]/notebook-[UNIQUE_NAME] diff --git a/acceptance/bundle/generate/python_job_and_deploy/output.txt b/acceptance/bundle/generate/python_job_and_deploy/output.txt index 397d2049b36..1694a8cee86 100644 --- a/acceptance/bundle/generate/python_job_and_deploy/output.txt +++ b/acceptance/bundle/generate/python_job_and_deploy/output.txt @@ -1,10 +1,13 @@ === Upload notebook to a workspace path + >>> [CLI] workspace import /Workspace/Users/[USERNAME]/test_notebook-[UNIQUE_NAME].py --file test_notebook.py --format AUTO --overwrite -=== Create a job that references the notebookCreated job +=== Create a job that references the notebook +Created job === Generate bundle config from the job + >>> [CLI] bundle generate job --existing-job-id [JOB_ID] --key out --config-dir resources --source-dir src --force File successfully saved to src/test_notebook-[UNIQUE_NAME].py Job configuration successfully saved to resources/out.job.yml @@ -18,19 +21,23 @@ include: === Verify generated yaml has expected fields + === Deploy the generated bundle + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python_job_and_deploy-[UNIQUE_NAME]/default/files... Files: 5 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 0 unchanged === Destroy the deployed bundle + >>> [CLI] bundle destroy --auto-approve All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/python_job_and_deploy-[UNIQUE_NAME]/default Destroy: 0 deleted === Cleanup: delete the original job and notebook + >>> errcode [CLI] jobs delete [JOB_ID] >>> errcode [CLI] workspace delete /Workspace/Users/[USERNAME]/test_notebook-[UNIQUE_NAME] diff --git a/acceptance/bundle/git-metadata/script b/acceptance/bundle/git-metadata/script index b629bc5776a..8999f7133d4 100644 --- a/acceptance/bundle/git-metadata/script +++ b/acceptance/bundle/git-metadata/script @@ -1,4 +1,4 @@ -title "a repository with a remote: branch, commit and origin URL come from .git\n" +title "a repository with a remote: branch, commit and origin URL come from .git" mkdir withremote cd withremote cp ../databricks.yml . @@ -8,7 +8,7 @@ trace $CLI bundle validate -o json | jq .bundle.git cd .. rm -fr withremote -title "a malformed .git: the worktree root is still found, with no metadata\n" +title "a malformed .git: the worktree root is still found, with no metadata" mkdir broken cd broken cp ../databricks.yml . diff --git a/acceptance/bundle/hashing/new-cli-on-old-state/output.txt b/acceptance/bundle/hashing/new-cli-on-old-state/output.txt index c9d7ef18808..b9cf79a10f7 100644 --- a/acceptance/bundle/hashing/new-cli-on-old-state/output.txt +++ b/acceptance/bundle/hashing/new-cli-on-old-state/output.txt @@ -3,5 +3,6 @@ json.state.resources.dashboards.dashboard1.state.serialized_dashboard = "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview\",\"layout\":[{\"widget\":{\"name\":\"counter1\"}}]}]}\n"; === Current CLI reads the old state without a phantom update + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/hashing/new-cli-on-old-state/script b/acceptance/bundle/hashing/new-cli-on-old-state/script index 24bd173a560..ab6aa0f9f79 100644 --- a/acceptance/bundle/hashing/new-cli-on-old-state/script +++ b/acceptance/bundle/hashing/new-cli-on-old-state/script @@ -5,7 +5,7 @@ envsubst < databricks.yml.tmpl > databricks.yml cleanup() { $CLI bundle destroy --auto-approve &> LOG.destroy; } trap cleanup EXIT -title "Old CLI (predates hashing) deploys: state holds the full serialized_dashboard\n" +title "Old CLI (predates hashing) deploys: state holds the full serialized_dashboard" $CLI_1_16_1 bundle deploy &> LOG.deploy_old contains.py '!panic' '!internal error' < LOG.deploy_old > /dev/null print_state.py | gron.py | grep serialized_dashboard diff --git a/acceptance/bundle/hashing/old-cli-on-new-state/output.txt b/acceptance/bundle/hashing/old-cli-on-new-state/output.txt index bc7817f8633..71d79dd4bc9 100644 --- a/acceptance/bundle/hashing/old-cli-on-new-state/output.txt +++ b/acceptance/bundle/hashing/old-cli-on-new-state/output.txt @@ -3,7 +3,8 @@ json.state.resources.dashboards.dashboard1.state.serialized_dashboard = "sha256:[HASH]"; === Old CLI on the hashed state plans a drift update, not a recreate or crash + === The old CLI redeploys and writes full content back, so the drift is self-correcting json.state.resources.dashboards.dashboard1.state.serialized_dashboard = "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview\",\"layout\":[{\"widget\":{\"name\":\"counter1\"}}]}]}\n"; -=== A second old-CLI plan is a no-op \ No newline at end of file +=== A second old-CLI plan is a no-op diff --git a/acceptance/bundle/hashing/old-cli-on-new-state/script b/acceptance/bundle/hashing/old-cli-on-new-state/script index a2ffb0a6931..8f7965b46d3 100644 --- a/acceptance/bundle/hashing/old-cli-on-new-state/script +++ b/acceptance/bundle/hashing/old-cli-on-new-state/script @@ -5,7 +5,7 @@ envsubst < databricks.yml.tmpl > databricks.yml cleanup() { $CLI bundle destroy --auto-approve &> LOG.destroy; } trap cleanup EXIT -title "Current CLI deploys: serialized_dashboard is stored as a content hash\n" +title "Current CLI deploys: serialized_dashboard is stored as a content hash" $CLI bundle deploy &> LOG.deploy_new contains.py '!panic' '!internal error' < LOG.deploy_new > /dev/null print_state.py | gron.py | grep serialized_dashboard @@ -16,7 +16,7 @@ title "Old CLI on the hashed state plans a drift update, not a recreate or crash $CLI_1_16_1 bundle plan &> LOG.plan_old contains.py '!recreate' '!panic' '!internal error' < LOG.plan_old > /dev/null -title "The old CLI redeploys and writes full content back, so the drift is self-correcting\n" +title "The old CLI redeploys and writes full content back, so the drift is self-correcting" $CLI_1_16_1 bundle deploy &> LOG.deploy_old contains.py '!panic' '!internal error' < LOG.deploy_old > /dev/null print_state.py | gron.py | grep serialized_dashboard diff --git a/acceptance/bundle/includes/plan_multifile/script b/acceptance/bundle/includes/plan_multifile/script index 08350775a73..bac97d0af5d 100644 --- a/acceptance/bundle/includes/plan_multifile/script +++ b/acceptance/bundle/includes/plan_multifile/script @@ -5,7 +5,7 @@ # merge this PR introduces for load performance. trace $CLI bundle plan -title "job_a merged max_concurrent_runs from the override file\n" +title "job_a merged max_concurrent_runs from the override file" # The located warning is already asserted by the plan above; drop the duplicate here so # this step only shows the merged value. trace $CLI bundle validate -o json 2>/dev/null | jq '.resources.jobs.job_a.max_concurrent_runs' diff --git a/acceptance/bundle/integration_whl/base/output.txt b/acceptance/bundle/integration_whl/base/output.txt index e45eb854f37..bb4b4825d99 100644 --- a/acceptance/bundle/integration_whl/base/output.txt +++ b/acceptance/bundle/integration_whl/base/output.txt @@ -48,6 +48,7 @@ Got arguments: ['my_test_code', 'one', 'two'] === Make a change to code without version change and run the job again + >>> [CLI] bundle deploy Building python_artifact... Uploading dist/my_test_code-0.0.1-py3-none-any.whl... diff --git a/acceptance/bundle/integration_whl/interactive_cluster/output.txt b/acceptance/bundle/integration_whl/interactive_cluster/output.txt index e56247af31c..13493969b4c 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster/output.txt +++ b/acceptance/bundle/integration_whl/interactive_cluster/output.txt @@ -49,6 +49,7 @@ Got arguments: ['my_test_code', 'one', 'two'] === Make a change to code without version change and run the job again + >>> [CLI] bundle deploy Building python_artifact... Uploading dist/my_test_code-0.0.1-py3-none-any.whl... diff --git a/acceptance/bundle/integration_whl/interactive_cluster_declared_libs/output.txt b/acceptance/bundle/integration_whl/interactive_cluster_declared_libs/output.txt index 96612a33430..9ce1aa60c4d 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster_declared_libs/output.txt +++ b/acceptance/bundle/integration_whl/interactive_cluster_declared_libs/output.txt @@ -1,11 +1,14 @@ === Deploy the cluster (declares requests and urllib3) and the wheel job that targets it + OK: deployed === Run the job: its task installs the wheel and pypi 'six' on the shared cluster, out of band + OK: job ran === Plan: the job's out-of-band 'six' shows as a removal, the declared libraries are unchanged + >>> plan_pypi_libs { "libraries[pypi='six;']": { @@ -19,6 +22,7 @@ OK: job ran } === Remove the declared 'urllib3' and redeploy: uninstalls urllib3 and the job's libraries, keeps requests + >>> plan_pypi_libs { "libraries[pypi='six;']": { diff --git a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt index 160d5c1abe4..ae2419161eb 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt +++ b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt @@ -18,6 +18,7 @@ Got arguments: ['my_test_code', 'one', 'two'] === Make a change to code without version change and run the job again + >>> [CLI] bundle deploy -qq >>> [CLI] bundle run some_other_job diff --git a/acceptance/bundle/integration_whl/interactive_single_user/output.txt b/acceptance/bundle/integration_whl/interactive_single_user/output.txt index 33047159fbd..69136022bca 100644 --- a/acceptance/bundle/integration_whl/interactive_single_user/output.txt +++ b/acceptance/bundle/integration_whl/interactive_single_user/output.txt @@ -46,6 +46,7 @@ Run URL: [DATABRICKS_URL]/jobs/[SOME_OTHER_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID [TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS === Make a change to code without version change and run the job again + >>> [CLI] bundle deploy -qq >>> [CLI] bundle run some_other_job diff --git a/acceptance/bundle/integration_whl/serverless/output.txt b/acceptance/bundle/integration_whl/serverless/output.txt index 99dfccb17a7..6ed9362e77c 100644 --- a/acceptance/bundle/integration_whl/serverless/output.txt +++ b/acceptance/bundle/integration_whl/serverless/output.txt @@ -19,6 +19,7 @@ Got arguments: ['my_test_code'] === Make a change to code without version change and run the job again + >>> [CLI] bundle deploy Building my_test_code... Uploading dist/my_test_code-0.0.1-py3-none-any.whl... diff --git a/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt b/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt index 1da7e8fcce6..ec0f9235d44 100644 --- a/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt +++ b/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt @@ -19,6 +19,7 @@ Got arguments: ['my_test_code'] === Make a change to code without version change and run the job again + >>> [CLI] bundle deploy Building my_test_code... Uploading .databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl... diff --git a/acceptance/bundle/integration_whl/serverless_extras/output.txt b/acceptance/bundle/integration_whl/serverless_extras/output.txt index c9c5ac6554d..ab1f5b88966 100644 --- a/acceptance/bundle/integration_whl/serverless_extras/output.txt +++ b/acceptance/bundle/integration_whl/serverless_extras/output.txt @@ -8,6 +8,7 @@ Files: 13 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === The environment dependency keeps its [train] extras suffix + >>> [CLI] jobs get [SOME_OTHER_JOB_ID] [ "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl[train]" diff --git a/acceptance/bundle/integration_whl/serverless_extras_dynamic_version/output.txt b/acceptance/bundle/integration_whl/serverless_extras_dynamic_version/output.txt index 61f8f061534..a065cf7c485 100644 --- a/acceptance/bundle/integration_whl/serverless_extras_dynamic_version/output.txt +++ b/acceptance/bundle/integration_whl/serverless_extras_dynamic_version/output.txt @@ -8,6 +8,7 @@ Files: 13 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === The environment dependency keeps its [train] extras suffix + >>> [CLI] jobs get [SOME_OTHER_JOB_ID] [ "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl[train]" diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/output.txt b/acceptance/bundle/libraries/outside_of_bundle_root/output.txt index 43e860f5d8e..e7f1bc13643 100644 --- a/acceptance/bundle/libraries/outside_of_bundle_root/output.txt +++ b/acceptance/bundle/libraries/outside_of_bundle_root/output.txt @@ -34,6 +34,7 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Check that the job libraries are uploaded and the path is correct in the job + >>> cat [OUTPUT_DIR]/out.requests.txt [ { diff --git a/acceptance/bundle/lifecycle/depends-on/output.txt b/acceptance/bundle/lifecycle/depends-on/output.txt index 494c0b523ee..530a92f52fb 100644 --- a/acceptance/bundle/lifecycle/depends-on/output.txt +++ b/acceptance/bundle/lifecycle/depends-on/output.txt @@ -1,5 +1,6 @@ === Plan records the edges + >>> [CLI] bundle plan -o json { "resources.jobs.a_job": [ @@ -38,6 +39,7 @@ } === Deploy creates dependencies first + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created jobs.a_job @@ -53,6 +55,7 @@ Resources: 6 created, 0 changed, 0 deleted, 0 unchanged POST /api/2.2/jobs/create a_job === Destroy deletes dependents first + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.a_job diff --git a/acceptance/bundle/local_state_staleness/output.txt b/acceptance/bundle/local_state_staleness/output.txt index 133b867061c..58690a3d413 100644 --- a/acceptance/bundle/local_state_staleness/output.txt +++ b/acceptance/bundle/local_state_staleness/output.txt @@ -1,5 +1,6 @@ === Step 1: Deploy bundle A + >>> [CLI] bundle deploy --force-lock --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/local-state-staleness-test-[UNIQUE_NAME]/files... Created jobs.test_job @@ -7,12 +8,14 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Step 2: Deploy bundle B + >>> [CLI] bundle deploy --force-lock --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/local-state-staleness-test-[UNIQUE_NAME]/files... Files: 2 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged === Step 3: Deploy bundle A again (should use remote state) + >>> [CLI] bundle deploy --force-lock --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/local-state-staleness-test-[UNIQUE_NAME]/files... Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/local_state_staleness/script b/acceptance/bundle/local_state_staleness/script index 90faf6de288..eaa5b1751eb 100644 --- a/acceptance/bundle/local_state_staleness/script +++ b/acceptance/bundle/local_state_staleness/script @@ -18,7 +18,7 @@ cp databricks.yml hello_world.py "$BUNDLE_A_DIR/" cp databricks.yml hello_world.py "$BUNDLE_B_DIR/" cleanup() { - title "Cleanup - destroy bundle A (bundle B does not have an active deployment)\n" + title "Cleanup - destroy bundle A (bundle B does not have an active deployment)" cd "$BUNDLE_A_DIR" || exit trace "$CLI" bundle destroy --auto-approve @@ -40,6 +40,6 @@ title "Step 3: Deploy bundle A again (should use remote state)" cd "$BUNDLE_A_DIR" trace "$CLI" bundle deploy --force-lock --auto-approve -title "Step 4: Verify only one job exists\n" +title "Step 4: Verify only one job exists" cd "$BUNDLE_A_DIR" "$CLI" jobs list -o json | jq -r --arg name "test-job-basic-${UNIQUE_NAME}" '.[] | select(.settings.name == $name) | {name: .settings.name, id: .job_id}' diff --git a/acceptance/bundle/migrate/auto/apply-failure/output.txt b/acceptance/bundle/migrate/auto/apply-failure/output.txt index be5d6f0a284..5e9de1e49f7 100644 --- a/acceptance/bundle/migrate/auto/apply-failure/output.txt +++ b/acceptance/bundle/migrate/auto/apply-failure/output.txt @@ -1,10 +1,12 @@ === Replay a prior terraform deployment + >>> replay_tfstate.py terraform-requests.json terraform.tfstate >>> update_file.py databricks.yml Test Auto-Migrate Apply Failure Test Auto-Migrate Apply Failure (updated) === Migrating deploy: migration commits (point of no return), then the job apply fails + >>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... @@ -36,6 +38,7 @@ ID Type Language Path [OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup === Retry finishes the apply on the already-migrated direct state (no re-migration) + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Updated jobs.test_job diff --git a/acceptance/bundle/migrate/auto/apply-failure/script b/acceptance/bundle/migrate/auto/apply-failure/script index 78b838fa835..1cb3fec5601 100644 --- a/acceptance/bundle/migrate/auto/apply-failure/script +++ b/acceptance/bundle/migrate/auto/apply-failure/script @@ -18,7 +18,7 @@ trace DATABRICKS_BUNDLE_ENGINE=direct errcode $CLI bundle deploy trace print_migration_telemetry rm -f "$OUT_REQUESTS" -title "On direct despite the failed apply: resources.json present, terraform state retired\n" +title "On direct despite the failed apply: resources.json present, terraform state retired" trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)' trace MSYS_NO_PATHCONV=1 $CLI workspace list "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/test-bundle/default/state" @@ -27,6 +27,6 @@ trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy trace print_migration_telemetry rm -f "$OUT_REQUESTS" -title "Local state remains direct\n" +title "Local state remains direct" trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)' trace MSYS_NO_PATHCONV=1 $CLI workspace list "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/test-bundle/default/state" diff --git a/acceptance/bundle/migrate/auto/artifacts/output.txt b/acceptance/bundle/migrate/auto/artifacts/output.txt index 0656827cd2d..5266dc7ed21 100644 --- a/acceptance/bundle/migrate/auto/artifacts/output.txt +++ b/acceptance/bundle/migrate/auto/artifacts/output.txt @@ -1,8 +1,10 @@ === Replay a prior terraform deployment + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Redeploy with direct default: auto-migration runs after the build, so library refs are resolved + >>> [CLI] bundle deploy Warn: unable to get spark version for cluster 0717-132531-5opeqon1, err: Not Found Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). @@ -26,6 +28,7 @@ ID Type Language Path [OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup === Migrated state's library path (should be the resolved remote path, not the local glob) + >>> print_state.py [ { @@ -34,6 +37,7 @@ ID Type Language Path ] === Subsequent deploy is a no-op on direct + >>> [CLI] bundle deploy Warn: unable to get spark version for cluster 0717-132531-5opeqon1, err: Not Found Uploading dist/my_test_code-0.0.1-py3-none-any.whl... diff --git a/acceptance/bundle/migrate/auto/artifacts/script b/acceptance/bundle/migrate/auto/artifacts/script index b351121e54a..5ee2f920b8f 100644 --- a/acceptance/bundle/migrate/auto/artifacts/script +++ b/acceptance/bundle/migrate/auto/artifacts/script @@ -6,7 +6,7 @@ trace replay_tfstate.py terraform-requests.json terraform.tfstate title "Redeploy with direct default: auto-migration runs after the build, so library refs are resolved" trace $CLI bundle deploy -title "Local state is now direct\n" +title "Local state is now direct" trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)' trace MSYS_NO_PATHCONV=1 $CLI workspace list "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/test-bundle/default/state" diff --git a/acceptance/bundle/migrate/auto/clean/output.txt b/acceptance/bundle/migrate/auto/clean/output.txt index e0cb4f41aaa..8adec4f3196 100644 --- a/acceptance/bundle/migrate/auto/clean/output.txt +++ b/acceptance/bundle/migrate/auto/clean/output.txt @@ -1,8 +1,10 @@ === Replay a prior terraform deployment + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Opt in to direct engine via config and redeploy: auto-migration triggers + >>> update_file.py databricks.yml name: test-bundle name: test-bundle engine: direct @@ -29,10 +31,12 @@ resources.json: remote direct state serial=3 lineage="[UUID]" [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=3 lineage="[UUID]" === Subsequent plan under direct is a no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Subsequent deploy under direct is a no-op + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 0 uploaded, 0 deleted @@ -41,6 +45,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged >>> print_migration_telemetry === Modify the config — plan shows the pending update + >>> update_file.py databricks.yml Test Auto-Migrate Job Test Auto-Migrate Job (renamed) >>> [CLI] bundle plan diff --git a/acceptance/bundle/migrate/auto/clean/script b/acceptance/bundle/migrate/auto/clean/script index 695759c1760..9ccce91996f 100644 --- a/acceptance/bundle/migrate/auto/clean/script +++ b/acceptance/bundle/migrate/auto/clean/script @@ -13,10 +13,10 @@ trace $CLI bundle deploy trace print_migration_telemetry rm -f "$OUT_REQUESTS" -title "Direct-engine state is now in place; terraform state is backed up\n" +title "Direct-engine state is now in place; terraform state is backed up" trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)' -title "Remote state also reflects the migration (force-pull surfaces workspace state)\n" +title "Remote state also reflects the migration (force-pull surfaces workspace state)" trace $CLI bundle debug states --force-pull title "Subsequent plan under direct is a no-op" @@ -32,7 +32,7 @@ trace update_file.py databricks.yml "Test Auto-Migrate Job" "Test Auto-Migrate J trace $CLI bundle plan $CLI bundle plan -o json > out.plan_update.json -title "Deploy picks up the update (with and without --plan)\n" +title "Deploy picks up the update (with and without --plan)" $CLI bundle deploy $(readplanarg out.plan_update.json) trace print_migration_telemetry rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/auto/default/output.txt b/acceptance/bundle/migrate/auto/default/output.txt index 30a5dd774e6..39a12c7eae6 100644 --- a/acceptance/bundle/migrate/auto/default/output.txt +++ b/acceptance/bundle/migrate/auto/default/output.txt @@ -1,8 +1,10 @@ === Replay a prior terraform deployment + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Summary works with engine: terraform and shows the deployed job's ID and URL + >>> update_file.py databricks.yml {name: test-bundle} {name: test-bundle, engine: terraform} >>> [CLI] bundle summary @@ -48,6 +50,7 @@ Notice: migrating your bundle to direct deployment engine (https://github.com/da >>> update_file.py databricks.yml {name: test-bundle, engine: terraform} {name: test-bundle} === Redeploy with nothing set: direct is the default, so auto-migration triggers + >>> [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... @@ -71,6 +74,7 @@ resources.json: remote direct state serial=3 lineage="[UUID]" [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=3 lineage="[UUID]" === Subsequent deploy is a no-op on direct + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 0 uploaded, 0 deleted @@ -79,6 +83,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged >>> print_migration_telemetry === Destroy the migrated bundle leaves no state behind, local or remote + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.test_job diff --git a/acceptance/bundle/migrate/auto/default/script b/acceptance/bundle/migrate/auto/default/script index e5a34df7e03..4f5847df312 100644 --- a/acceptance/bundle/migrate/auto/default/script +++ b/acceptance/bundle/migrate/auto/default/script @@ -14,7 +14,7 @@ trace update_file.py databricks.yml "{name: test-bundle}" "{name: test-bundle, e trace $CLI bundle summary trace $CLI bundle summary -o json | jq '{engine: .bundle.engine, id: .resources.jobs.test_job.id, url: .resources.jobs.test_job.url}' -title "Summary does not migrate: no state writes, terraform state is left untouched\n" +title "Summary does not migrate: no state writes, terraform state is left untouched" trace print_requests.py trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)' trace update_file.py databricks.yml "{name: test-bundle, engine: terraform}" "{name: test-bundle}" @@ -24,10 +24,10 @@ trace $CLI bundle deploy trace print_migration_telemetry rm -f "$OUT_REQUESTS" -title "Direct-engine state is now in place; terraform state is backed up\n" +title "Direct-engine state is now in place; terraform state is backed up" trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)' -title "Remote state also reflects the migration (force-pull surfaces workspace state)\n" +title "Remote state also reflects the migration (force-pull surfaces workspace state)" trace $CLI bundle debug states --force-pull title "Subsequent deploy is a no-op on direct" diff --git a/acceptance/bundle/migrate/auto/delete-all/output.txt b/acceptance/bundle/migrate/auto/delete-all/output.txt index f03f24ba55e..e034c5ab1bf 100644 --- a/acceptance/bundle/migrate/auto/delete-all/output.txt +++ b/acceptance/bundle/migrate/auto/delete-all/output.txt @@ -1,8 +1,10 @@ === Replay a prior terraform deployment + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Remove all resources, then deploy on direct: migrate and delete + >>> [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... @@ -12,5 +14,6 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 0 changed, 1 deleted, 0 unchanged === Follow-up plan is a clean no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/migrate/auto/delete/output.txt b/acceptance/bundle/migrate/auto/delete/output.txt index 7d6a7b04dd2..0f3b1550db7 100644 --- a/acceptance/bundle/migrate/auto/delete/output.txt +++ b/acceptance/bundle/migrate/auto/delete/output.txt @@ -1,8 +1,10 @@ === Replay a prior terraform deployment + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Drop one job from config, then deploy on direct: migrate two, delete the dropped one + >>> [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... @@ -12,5 +14,6 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 0 changed, 1 deleted, 1 unchanged === Follow-up plan is a clean no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/migrate/auto/depends-on/output.txt b/acceptance/bundle/migrate/auto/depends-on/output.txt index bfba175139e..48a6e3778f0 100644 --- a/acceptance/bundle/migrate/auto/depends-on/output.txt +++ b/acceptance/bundle/migrate/auto/depends-on/output.txt @@ -1,8 +1,10 @@ === Replay a prior terraform deployment + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Deploy migrates to direct and honors lifecycle.depends_on + >>> [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... @@ -15,6 +17,7 @@ Resources: 1 created, 0 changed, 0 deleted, 1 unchanged direct_migrated_via_default true === The edge is recorded in the plan + >>> [CLI] bundle plan -o json { "resources.jobs.other_job": [ diff --git a/acceptance/bundle/migrate/auto/destroy/output.txt b/acceptance/bundle/migrate/auto/destroy/output.txt index 7022244297c..c73f07cc98c 100644 --- a/acceptance/bundle/migrate/auto/destroy/output.txt +++ b/acceptance/bundle/migrate/auto/destroy/output.txt @@ -1,8 +1,10 @@ === Replay a prior terraform deployment + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Destroy without approval: migration prepared in memory then dropped, stays on terraform + >>> musterr [CLI] bundle destroy Error: this command will destroy all resources deployed by this bundle, including workspace files in the deployment directory. Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. @@ -16,6 +18,7 @@ ID Type Language Path [OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate === Destroy on direct: migrate then destroy + >>> [CLI] bundle destroy --auto-approve Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). The following resources will be deleted: @@ -27,6 +30,7 @@ Migrated 1 resource to direct deployment engine. Destroy: 1 deleted === Destroy leaves no state behind: no local state files, remote deployment removed + >>> find.py \.databricks/.*(\.tfstate|resources.*\.json|\.wal) >>> MSYS_NO_PATHCONV=1 musterr [CLI] workspace list /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state diff --git a/acceptance/bundle/migrate/auto/direct-only-envvar/output.txt b/acceptance/bundle/migrate/auto/direct-only-envvar/output.txt index b069e473748..e6cdd138f09 100644 --- a/acceptance/bundle/migrate/auto/direct-only-envvar/output.txt +++ b/acceptance/bundle/migrate/auto/direct-only-envvar/output.txt @@ -1,11 +1,14 @@ === Replay a prior terraform deployment + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Add a direct-only resource to the config + >>> update_file.py databricks.yml #instance_pools instance_pools === Opt in via env var only: deploy migrates the state and creates the direct-only resource + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... @@ -18,6 +21,7 @@ Resources: 1 created, 0 changed, 0 deleted, 1 unchanged direct_migrated_via_env true === Redeploy is a no-op: state is already on the direct engine + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -29,5 +33,6 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //instance-pools/create === No further actions planned + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/migrate/auto/direct-only/output.txt b/acceptance/bundle/migrate/auto/direct-only/output.txt index 10fcf4b0bc6..b432c5872e7 100644 --- a/acceptance/bundle/migrate/auto/direct-only/output.txt +++ b/acceptance/bundle/migrate/auto/direct-only/output.txt @@ -1,14 +1,17 @@ === Replay a prior terraform deployment + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Add a direct-only resource and opt in to the direct engine via config + >>> update_file.py databricks.yml #instance_pools instance_pools >>> update_file.py databricks.yml name: test-bundle name: test-bundle engine: direct === Plan runs on the direct engine (state migrated in memory), so the direct-only resource is planned + >>> [CLI] bundle plan Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). create instance_pools.pool @@ -23,6 +26,7 @@ json.plan.resources.instance_pools.pool.new_state.value.instance_pool_name = "My json.plan.resources.instance_pools.pool.new_state.value.node_type_id = "Standard_DS3_v2"; === Plan does not commit state change, terraform stays + >>> find.py \.databricks/.*(\.tfstate|resources.*\.json|\.wal) .databricks/bundle/default/terraform/terraform.tfstate @@ -31,6 +35,7 @@ ID Type Language Path [OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate === Deploy commits the migration and creates the direct-only resource + >>> [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... @@ -43,6 +48,7 @@ Resources: 1 created, 0 changed, 0 deleted, 1 unchanged direct_migrated_via_config true === Redeploy is a no-op: state is already on the direct engine + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -54,5 +60,6 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //instance-pools/create === No further actions planned + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/migrate/auto/dms-migration/output.txt b/acceptance/bundle/migrate/auto/dms-migration/output.txt index 3642c1759fd..f3dc7d8af2e 100644 --- a/acceptance/bundle/migrate/auto/dms-migration/output.txt +++ b/acceptance/bundle/migrate/auto/dms-migration/output.txt @@ -1,8 +1,10 @@ === Replay a prior terraform deployment (captured from a real terraform run) + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Terraform state migrates to file-backed direct state with shadow history + >>> DATABRICKS_BUNDLE_ENGINE=direct DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... diff --git a/acceptance/bundle/migrate/auto/empty-plan-missing/output.txt b/acceptance/bundle/migrate/auto/empty-plan-missing/output.txt index 795b9d599d4..1d2db3f3872 100644 --- a/acceptance/bundle/migrate/auto/empty-plan-missing/output.txt +++ b/acceptance/bundle/migrate/auto/empty-plan-missing/output.txt @@ -1,5 +1,6 @@ === deploy --plan with a missing plan file fails after the empty migration prepared the direct state + >>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy --plan does-not-exist.json Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Error: reading plan file: open does-not-exist.json: no such file or directory @@ -8,5 +9,6 @@ Error: reading plan file: open does-not-exist.json: no such file or directory Exit code: 1 === Terraform state is untouched - the empty migration was never committed or swept + >>> find.py \.databricks/.*(\.tfstate|resources.*\.json|\.wal) .databricks/bundle/default/terraform/terraform.tfstate diff --git a/acceptance/bundle/migrate/auto/empty-tfstate/output.txt b/acceptance/bundle/migrate/auto/empty-tfstate/output.txt index 54447ea8ecc..c3f67e7e045 100644 --- a/acceptance/bundle/migrate/auto/empty-tfstate/output.txt +++ b/acceptance/bundle/migrate/auto/empty-tfstate/output.txt @@ -1,5 +1,6 @@ === Deploy with bundle.engine: direct and an empty terraform state — migrate (0 resources) + >>> [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... @@ -38,6 +39,7 @@ resources.json: remote direct state serial=2 lineage="test-lineage" [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=2 lineage="test-lineage" === Next deploy uses direct engine by default (no more mismatch warning, no re-migration) + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 0 uploaded, 0 deleted diff --git a/acceptance/bundle/migrate/auto/empty-tfstate/script b/acceptance/bundle/migrate/auto/empty-tfstate/script index 225092954a3..4d73d5915ad 100644 --- a/acceptance/bundle/migrate/auto/empty-tfstate/script +++ b/acceptance/bundle/migrate/auto/empty-tfstate/script @@ -12,16 +12,16 @@ EOF title "Deploy with bundle.engine: direct and an empty terraform state — migrate (0 resources)" trace $CLI bundle deploy 2>&1 | contains.py 'Migrated 0 resources to direct deployment engine.' -title "Empty state migrated: resources.json written, local terraform state backed up\n" +title "Empty state migrated: resources.json written, local terraform state backed up" trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)' -title "Migration is recorded in telemetry as via-config\n" +title "Migration is recorded in telemetry as via-config" trace print_migration_telemetry -title "The remote had no terraform state, so nothing terraform-related is backed up remotely\n" +title "The remote had no terraform state, so nothing terraform-related is backed up remotely" trace print_requests.py //workspace-files/import-file //workspace/delete --sort | jq '.method + " " + .path' | contains.py '!terraform.tfstate' -title "debug states confirms no terraform.tfstate anywhere; only the local .backup remains\n" +title "debug states confirms no terraform.tfstate anywhere; only the local .backup remains" trace $CLI bundle debug states --force-pull title "Next deploy uses direct engine by default (no more mismatch warning, no re-migration)" diff --git a/acceptance/bundle/migrate/auto/envvar/output.txt b/acceptance/bundle/migrate/auto/envvar/output.txt index e67cc31e1c9..1e7da50ffcc 100644 --- a/acceptance/bundle/migrate/auto/envvar/output.txt +++ b/acceptance/bundle/migrate/auto/envvar/output.txt @@ -1,8 +1,10 @@ === Replay a prior terraform deployment + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Opt in via env var only (no bundle.engine in config): auto-migration triggers + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... diff --git a/acceptance/bundle/migrate/auto/envvar/script b/acceptance/bundle/migrate/auto/envvar/script index f704b02f97c..77784eed11c 100644 --- a/acceptance/bundle/migrate/auto/envvar/script +++ b/acceptance/bundle/migrate/auto/envvar/script @@ -13,10 +13,10 @@ trace print_migration_telemetry trace print_requests.py | grep -o 'engine/[a-z]*' | sort -u rm -f "$OUT_REQUESTS" -title "Direct-engine state is now in place; terraform state is backed up\n" +title "Direct-engine state is now in place; terraform state is backed up" trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)' -title "Remote state also reflects the migration (force-pull surfaces workspace state)\n" +title "Remote state also reflects the migration (force-pull surfaces workspace state)" trace $CLI bundle debug states --force-pull rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/auto/new-resource-ref/output.txt b/acceptance/bundle/migrate/auto/new-resource-ref/output.txt index e3c122f31c1..7ddc2fe273a 100644 --- a/acceptance/bundle/migrate/auto/new-resource-ref/output.txt +++ b/acceptance/bundle/migrate/auto/new-resource-ref/output.txt @@ -1,9 +1,12 @@ === Replay a prior terraform deployment of a job only + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Add a new alert and a job task that references it + === Plan shows the new alert and the job update (the terraform state is migrated in memory) + >>> [CLI] bundle plan -o json Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). { @@ -147,6 +150,7 @@ Notice: migrating your bundle to direct deployment engine (https://github.com/da } === Deploy migrates to direct, creates the alert and updates the job + >>> [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... @@ -157,11 +161,13 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 1 changed, 0 deleted, 0 unchanged === The job update carries the new alert id and timeout + >>> print_requests.py //api/2.2/jobs/reset json.body.new_settings.tasks[0].alert_task.alert_id = "[NEW_ALERT_ID]"; json.body.new_settings.timeout_seconds = 300; === Redeploy is a no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/migrate/auto/pipeline-storage-change/output.txt b/acceptance/bundle/migrate/auto/pipeline-storage-change/output.txt index 7e5d258fdf0..1302bbdb3ef 100644 --- a/acceptance/bundle/migrate/auto/pipeline-storage-change/output.txt +++ b/acceptance/bundle/migrate/auto/pipeline-storage-change/output.txt @@ -1,8 +1,10 @@ === Replay the prior terraform deployment (captured from a real terraform run) + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Change immutable storage. Without --auto-approve the recreate is refused + >>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy Warning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes. at resources.pipelines.my_pipeline.development @@ -32,6 +34,7 @@ direct_migrate_recreate_planned true .databricks/bundle/default/terraform/terraform.tfstate === With --auto-approve the recreate is applied on direct + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy --auto-approve Warning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes. at resources.pipelines.my_pipeline.development @@ -51,11 +54,13 @@ Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 0 unchanged === Local state is now direct + >>> find.py \.databricks/.*(\.tfstate|resources.*\.json|\.wal) .databricks/bundle/default/resources.json .databricks/bundle/default/terraform/terraform.tfstate.backup === Cleanup + >>> [CLI] bundle destroy --auto-approve Warning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes. at resources.pipelines.my_pipeline.development diff --git a/acceptance/bundle/migrate/auto/plan-failure/output.txt b/acceptance/bundle/migrate/auto/plan-failure/output.txt index 343ba610c1d..ea36c06bc7c 100644 --- a/acceptance/bundle/migrate/auto/plan-failure/output.txt +++ b/acceptance/bundle/migrate/auto/plan-failure/output.txt @@ -1,10 +1,12 @@ === Replay a prior terraform deployment (captured from a real terraform run) + >>> replay_tfstate.py terraform-requests.json terraform.tfstate >>> print_migration_telemetry === Inject GET failure so the migration plan check fails: migration hard-errors (no terraform fallback) + >>> DATABRICKS_BUNDLE_ENGINE=direct musterr [CLI] bundle deploy Warn: migration to direct: cannot plan resources.jobs.test_job: reading id="[TEST_JOB_ID]": Fault injected by test. (403 INJECTED): Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/get?job_id=[TEST_JOB_ID] HTTP Status: 403 Forbidden @@ -19,6 +21,7 @@ The Terraform deployment engine was removed in Databricks CLI v1.20.0. Auto-migr direct_migrate_plan_error true === Local state was NOT rewritten (still terraform) + >>> find.py \.databricks/.*(\.tfstate|resources.*\.json|\.wal) .databricks/bundle/default/terraform/terraform.tfstate @@ -27,6 +30,7 @@ ID Type Language Path [OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate === Retry: plan check passes this time, migration succeeds + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... @@ -38,6 +42,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged direct_migrated_via_env true === Local state is now direct + >>> find.py \.databricks/.*(\.tfstate|resources.*\.json|\.wal) .databricks/bundle/default/resources.json .databricks/bundle/default/terraform/terraform.tfstate.backup diff --git a/acceptance/bundle/migrate/auto/plan-missing/output.txt b/acceptance/bundle/migrate/auto/plan-missing/output.txt index 74d958908d0..c1b00e554ed 100644 --- a/acceptance/bundle/migrate/auto/plan-missing/output.txt +++ b/acceptance/bundle/migrate/auto/plan-missing/output.txt @@ -1,8 +1,10 @@ === Replay a prior terraform deployment (captured from a real terraform run) + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === deploy --plan with a missing plan file fails after the migration prepared the direct state + >>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy --plan does-not-exist.json Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Error: reading plan file: open does-not-exist.json: no such file or directory @@ -11,6 +13,7 @@ Error: reading plan file: open does-not-exist.json: no such file or directory Exit code: 1 === State is still terraform - the migration was never committed + >>> find.py \.databricks/.*(\.tfstate|resources.*\.json|\.wal) .databricks/bundle/default/terraform/terraform.tfstate @@ -19,6 +22,7 @@ ID Type Language Path [OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate === A plain deploy migrates the still-terraform state to direct + >>> [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... diff --git a/acceptance/bundle/migrate/auto/push-failure/output.txt b/acceptance/bundle/migrate/auto/push-failure/output.txt index 6e9a1c61b78..acf72a60c36 100644 --- a/acceptance/bundle/migrate/auto/push-failure/output.txt +++ b/acceptance/bundle/migrate/auto/push-failure/output.txt @@ -1,8 +1,10 @@ === Replay a prior terraform deployment (captured from a real terraform run) + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Opt in via env var; inject 403 on the resources.json push so the deploy fails + >>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... @@ -31,6 +33,7 @@ ID Type Language Path [OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate === Retry: the local direct state is used (no re-migration) and the push completes + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 0 uploaded, 0 deleted diff --git a/acceptance/bundle/migrate/auto/push-failure/script b/acceptance/bundle/migrate/auto/push-failure/script index da3ffaad3d9..00e611e1391 100644 --- a/acceptance/bundle/migrate/auto/push-failure/script +++ b/acceptance/bundle/migrate/auto/push-failure/script @@ -15,7 +15,7 @@ trace DATABRICKS_BUNDLE_ENGINE=direct errcode $CLI bundle deploy trace print_migration_telemetry rm -f "$OUT_REQUESTS" -title "Local state is direct (the migration wrote it) though the remote push did not land\n" +title "Local state is direct (the migration wrote it) though the remote push did not land" trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)' trace MSYS_NO_PATHCONV=1 $CLI workspace list "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/test-bundle/default/state" @@ -24,7 +24,7 @@ trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy trace print_migration_telemetry rm -f "$OUT_REQUESTS" -title "Retry recovered: local state is direct (the outranked local terraform.tfstate lingers)\n" +title "Retry recovered: local state is direct (the outranked local terraform.tfstate lingers)" trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)' trace MSYS_NO_PATHCONV=1 $CLI workspace list "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/test-bundle/default/state" diff --git a/acceptance/bundle/migrate/auto/registered-model-catalog-change/output.txt b/acceptance/bundle/migrate/auto/registered-model-catalog-change/output.txt index 40f559184ca..57d2b4a1da9 100644 --- a/acceptance/bundle/migrate/auto/registered-model-catalog-change/output.txt +++ b/acceptance/bundle/migrate/auto/registered-model-catalog-change/output.txt @@ -1,8 +1,10 @@ === Replay the prior terraform deployment (captured from a real terraform run) + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Move the model to another catalog and deploy on direct + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Warn: migration to direct: registered_models.my_model: catalog_name differs between config ("othercat_[UNIQUE_NAME]") and terraform state ("main"); this resource will be recreated. Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). @@ -17,12 +19,14 @@ direct_migrate_recreate_planned true direct_migrated_via_env true === Redeploy on direct: the model now matches config, so the plan is a clean no-op + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged === The model exists in the new catalog and the local state is direct + >>> [CLI] registered-models get othercat_[UNIQUE_NAME].default.mymodel-[UNIQUE_NAME] { "full_name": "othercat_[UNIQUE_NAME].default.mymodel-[UNIQUE_NAME]" @@ -33,6 +37,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged .databricks/bundle/default/terraform/terraform.tfstate.backup === Cleanup + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.registered_models.my_model diff --git a/acceptance/bundle/migrate/auto/registered-model-rename/output.txt b/acceptance/bundle/migrate/auto/registered-model-rename/output.txt index fbad32d54f3..a81db3e5b61 100644 --- a/acceptance/bundle/migrate/auto/registered-model-rename/output.txt +++ b/acceptance/bundle/migrate/auto/registered-model-rename/output.txt @@ -1,8 +1,10 @@ === Replay the prior terraform deployment (captured from a real terraform run) + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Rename the model (an immutable id field) and deploy on direct + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Warn: migration to direct: registered_models.my_model: name differs between config ("themodel-[UNIQUE_NAME]-renamed") and terraform state ("themodel-[UNIQUE_NAME]"); this resource will be recreated. Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). @@ -17,12 +19,14 @@ direct_migrate_recreate_planned true direct_migrated_via_env true === Redeploy on direct: the model now matches config, so the plan is a clean no-op + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 2 unchanged === The model exists under the new name and the local state is direct + >>> [CLI] registered-models get main.mschema-[UNIQUE_NAME].themodel-[UNIQUE_NAME]-renamed { "full_name": "main.mschema-[UNIQUE_NAME].themodel-[UNIQUE_NAME]-renamed" @@ -33,6 +37,7 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged .databricks/bundle/default/terraform/terraform.tfstate.backup === Cleanup + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.registered_models.my_model diff --git a/acceptance/bundle/migrate/auto/run-no-commit/output.txt b/acceptance/bundle/migrate/auto/run-no-commit/output.txt index 87ab33b2c30..067d03c70fd 100644 --- a/acceptance/bundle/migrate/auto/run-no-commit/output.txt +++ b/acceptance/bundle/migrate/auto/run-no-commit/output.txt @@ -1,8 +1,10 @@ === Replay a prior terraform deployment + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === bundle run on direct migrates the terraform state IN MEMORY (no commit) to resolve the job + >>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle run test_job --no-wait Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Run URL: [DATABRICKS_URL]/jobs/[TEST_JOB_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] @@ -20,6 +22,7 @@ ID Type Language Path [OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate === bundle run works with engine: terraform: warns and runs the job from the terraform state + >>> update_file.py databricks.yml {name: test-bundle} {name: test-bundle, engine: terraform} >>> [CLI] bundle run test_job --no-wait diff --git a/acceptance/bundle/migrate/auto/run-no-commit/script b/acceptance/bundle/migrate/auto/run-no-commit/script index 2640ea4e235..e099abf2e1e 100644 --- a/acceptance/bundle/migrate/auto/run-no-commit/script +++ b/acceptance/bundle/migrate/auto/run-no-commit/script @@ -13,7 +13,7 @@ trace DATABRICKS_BUNDLE_ENGINE=direct errcode $CLI bundle run test_job --no-wait trace print_migration_telemetry rm -f "$OUT_REQUESTS" -title "Nothing durable written: still on terraform (local state files + remote), no direct state / migrating temp / WAL\n" +title "Nothing durable written: still on terraform (local state files + remote), no direct state / migrating temp / WAL" # find.py lists matching files sorted with forward slashes (cross-platform); the terraform provider # cache under .terraform/ is excluded by matching only state files (tfstate/.backup, resources*.json, WAL). trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)' diff --git a/acceptance/bundle/migrate/auto/schema-catalog-change/output.txt b/acceptance/bundle/migrate/auto/schema-catalog-change/output.txt index d49b7055852..e8f06c36d89 100644 --- a/acceptance/bundle/migrate/auto/schema-catalog-change/output.txt +++ b/acceptance/bundle/migrate/auto/schema-catalog-change/output.txt @@ -1,8 +1,10 @@ === Replay the prior terraform deployment (captured from a real terraform run) + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Move the schema to another catalog. Without --auto-approve the recreate is refused + >>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy Warn: migration to direct: schemas.my_schema: catalog_name differs between config ("othercat_[UNIQUE_NAME]") and terraform state ("main"); this resource will be recreated. Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). @@ -23,6 +25,7 @@ Exit code: 1 direct_migrate_recreate_planned true === With --auto-approve the recreate moves the schema to the new catalog on direct + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy --auto-approve Warn: migration to direct: schemas.my_schema: catalog_name differs between config ("othercat_[UNIQUE_NAME]") and terraform state ("main"); this resource will be recreated. Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). @@ -36,12 +39,14 @@ Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 0 unchanged === Redeploy on direct: the schema now matches config, so the plan is a clean no-op + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged === The schema exists in the new catalog and the local state is direct + >>> [CLI] schemas get othercat_[UNIQUE_NAME].myschema-[UNIQUE_NAME] { "full_name": "othercat_[UNIQUE_NAME].myschema-[UNIQUE_NAME]" @@ -52,6 +57,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged .databricks/bundle/default/terraform/terraform.tfstate.backup === Cleanup + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.schemas.my_schema diff --git a/acceptance/bundle/migrate/auto/schema-name-normalized/output.txt b/acceptance/bundle/migrate/auto/schema-name-normalized/output.txt index cdfe47510ee..fb8d112d4b0 100644 --- a/acceptance/bundle/migrate/auto/schema-name-normalized/output.txt +++ b/acceptance/bundle/migrate/auto/schema-name-normalized/output.txt @@ -1,8 +1,10 @@ === Replay the prior terraform deployment (captured from a real terraform run) + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Migrate to direct with the same config (mixed-case name unchanged) + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... @@ -11,10 +13,12 @@ Files: 2 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged === Follow-up plan on direct converges (the case-only difference is not a change) + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Cleanup + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.schemas.my_schema diff --git a/acceptance/bundle/migrate/auto/schema-rename/output.txt b/acceptance/bundle/migrate/auto/schema-rename/output.txt index 65e060f08f9..6acf51246ff 100644 --- a/acceptance/bundle/migrate/auto/schema-rename/output.txt +++ b/acceptance/bundle/migrate/auto/schema-rename/output.txt @@ -1,8 +1,10 @@ === Replay the prior terraform deployment (captured from a real terraform run) + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Rename the schema (an immutable id field). Without --auto-approve the recreate is refused + >>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy Warn: migration to direct: schemas.my_schema: name differs between config ("myschema-[UNIQUE_NAME]-renamed") and terraform state ("myschema-[UNIQUE_NAME]"); this resource will be recreated. Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). @@ -23,6 +25,7 @@ Exit code: 1 direct_migrate_recreate_planned true === With --auto-approve the recreate is applied on direct under the new name + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy --auto-approve Warn: migration to direct: schemas.my_schema: name differs between config ("myschema-[UNIQUE_NAME]-renamed") and terraform state ("myschema-[UNIQUE_NAME]"); this resource will be recreated. Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). @@ -36,12 +39,14 @@ Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 0 unchanged === Redeploy on direct: the schema now matches config, so the plan is a clean no-op + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged === The schema exists under the new name and the local state is direct + >>> [CLI] schemas get main.myschema-[UNIQUE_NAME]-renamed { "full_name": "main.myschema-[UNIQUE_NAME]-renamed" @@ -52,6 +57,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged .databricks/bundle/default/terraform/terraform.tfstate.backup === Cleanup + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.schemas.my_schema diff --git a/acceptance/bundle/migrate/auto/schema-storage-change/output.txt b/acceptance/bundle/migrate/auto/schema-storage-change/output.txt index bc4083f4457..8ba6441ad3b 100644 --- a/acceptance/bundle/migrate/auto/schema-storage-change/output.txt +++ b/acceptance/bundle/migrate/auto/schema-storage-change/output.txt @@ -1,10 +1,12 @@ === Replay the prior terraform deployment (captured from a real terraform run) + >>> replay_tfstate.py terraform-requests.json terraform.tfstate >>> print_migration_telemetry === Change immutable storage_root; without --auto-approve the recreate is refused + >>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... @@ -24,6 +26,7 @@ Exit code: 1 direct_migrate_recreate_planned true === The migration was not committed: the state is still terraform + >>> find.py \.databricks/.*(\.tfstate|resources.*\.json|\.wal) .databricks/bundle/default/terraform/terraform.tfstate @@ -33,6 +36,7 @@ ID Type Language Path [OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate === With --auto-approve the migration commits and the recreate is applied on direct + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy --auto-approve Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... @@ -49,6 +53,7 @@ direct_migrate_recreate_planned true direct_migrated_via_env true === Redeploy on direct: the schema now matches config, so the plan is a clean no-op + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 0 uploaded, 0 deleted diff --git a/acceptance/bundle/migrate/auto/schema-storage-normalized/output.txt b/acceptance/bundle/migrate/auto/schema-storage-normalized/output.txt index ccb8f6f49fa..e059a976b43 100644 --- a/acceptance/bundle/migrate/auto/schema-storage-normalized/output.txt +++ b/acceptance/bundle/migrate/auto/schema-storage-normalized/output.txt @@ -1,8 +1,10 @@ === Replay the prior terraform deployment (captured from a real terraform run) + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Migrate to direct with the same config (storage_root still has the trailing slash) + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... @@ -11,5 +13,6 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged === Follow-up plan on direct: does it converge or keep seeing a diff? + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/migrate/auto/tf-resource-type-references/output.txt b/acceptance/bundle/migrate/auto/tf-resource-type-references/output.txt index a75cf4019df..26367944d98 100644 --- a/acceptance/bundle/migrate/auto/tf-resource-type-references/output.txt +++ b/acceptance/bundle/migrate/auto/tf-resource-type-references/output.txt @@ -1,8 +1,10 @@ === Replay a prior terraform deployment + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Validation does not flag the reference + >>> [CLI] bundle validate Name: test-bundle Target: default @@ -13,12 +15,14 @@ Workspace: Validation OK! === Plan after auto-migration + >>> musterr [CLI] bundle plan Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Error: invalid dependency "${databricks_pipeline.src_pipeline.id}", no such node "" === Deploy after auto-migration + >>> musterr [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... diff --git a/acceptance/bundle/migrate/auto/tfbackup-failure/output.txt b/acceptance/bundle/migrate/auto/tfbackup-failure/output.txt index 8475d5c21a0..d82a65b908b 100644 --- a/acceptance/bundle/migrate/auto/tfbackup-failure/output.txt +++ b/acceptance/bundle/migrate/auto/tfbackup-failure/output.txt @@ -1,8 +1,10 @@ === Replay a prior terraform deployment (captured from a real terraform run) + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Opt in via env var; inject 403 on the DELETE of remote terraform.tfstate + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... @@ -29,6 +31,7 @@ ID Type Language Path [OBJECT_ID] FILE /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/terraform.tfstate.backup === Retry: deploys on already-migrated direct state + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 0 uploaded, 0 deleted diff --git a/acceptance/bundle/migrate/auto/tfbackup-failure/script b/acceptance/bundle/migrate/auto/tfbackup-failure/script index b2d2d9a6618..fb1f1c74cb9 100644 --- a/acceptance/bundle/migrate/auto/tfbackup-failure/script +++ b/acceptance/bundle/migrate/auto/tfbackup-failure/script @@ -16,7 +16,7 @@ trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy trace print_migration_telemetry rm -f "$OUT_REQUESTS" -title "Local state is now direct (migration succeeded with warning)\n" +title "Local state is now direct (migration succeeded with warning)" trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)' trace MSYS_NO_PATHCONV=1 $CLI workspace list "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/test-bundle/default/state" @@ -25,7 +25,7 @@ trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy trace print_migration_telemetry rm -f "$OUT_REQUESTS" -title "Local state remains direct\n" +title "Local state remains direct" trace find.py '\.databricks/.*(\.tfstate|resources.*\.json|\.wal)' rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/migrate/auto/volume-name-normalized/output.txt b/acceptance/bundle/migrate/auto/volume-name-normalized/output.txt index 38b0ebc578c..55b6296e16f 100644 --- a/acceptance/bundle/migrate/auto/volume-name-normalized/output.txt +++ b/acceptance/bundle/migrate/auto/volume-name-normalized/output.txt @@ -1,8 +1,10 @@ === Replay the prior terraform deployment (captured from a real terraform run) + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Migrate to direct with the same config (mixed-case name unchanged) + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... @@ -11,10 +13,12 @@ Files: 2 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged === Follow-up plan on direct converges (the case-only difference is not a change) + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Cleanup + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.volumes.my_volume diff --git a/acceptance/bundle/migrate/auto/volume-rename/output.txt b/acceptance/bundle/migrate/auto/volume-rename/output.txt index 762538efab5..90c63426aaa 100644 --- a/acceptance/bundle/migrate/auto/volume-rename/output.txt +++ b/acceptance/bundle/migrate/auto/volume-rename/output.txt @@ -1,8 +1,10 @@ === Replay the prior terraform deployment (captured from a real terraform run) + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Rename the volume (an updatable id field) and deploy on direct + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Warn: migration to direct: volumes.my_volume: name differs between config ("thevolume-[UNIQUE_NAME]-renamed") and terraform state ("thevolume-[UNIQUE_NAME]"); this resource will be renamed. Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). @@ -13,12 +15,14 @@ Files: 2 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged === Retry on direct: state now matches the backend, so the migration succeeds + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 2 unchanged === The volume exists under the new name and the local state is direct + >>> [CLI] volumes read main.pschema-[UNIQUE_NAME].thevolume-[UNIQUE_NAME]-renamed { "full_name": "main.pschema-[UNIQUE_NAME].thevolume-[UNIQUE_NAME]-renamed" @@ -29,6 +33,7 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged .databricks/bundle/default/terraform/terraform.tfstate.backup === Cleanup + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.schemas.parent diff --git a/acceptance/bundle/migrate/auto/volume-type-change/output.txt b/acceptance/bundle/migrate/auto/volume-type-change/output.txt index 40989888210..b60c63e771b 100644 --- a/acceptance/bundle/migrate/auto/volume-type-change/output.txt +++ b/acceptance/bundle/migrate/auto/volume-type-change/output.txt @@ -1,8 +1,10 @@ === Replay the prior terraform deployment (captured from a real terraform run) + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Change volume_type (immutable); without --auto-approve the recreate is refused + >>> DATABRICKS_BUNDLE_ENGINE=direct errcode [CLI] bundle deploy Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... diff --git a/acceptance/bundle/migrate/command/added-direct-only/output.txt b/acceptance/bundle/migrate/command/added-direct-only/output.txt index 2d6abe8d412..96bcd2bda87 100644 --- a/acceptance/bundle/migrate/command/added-direct-only/output.txt +++ b/acceptance/bundle/migrate/command/added-direct-only/output.txt @@ -1,11 +1,14 @@ === Replay a prior terraform deployment (instance_pools is commented out) + >>> replay_tfstate.py terraform-requests.json terraform.tfstate === Add a direct-only resource to the config without deploying it + >>> update_file.py databricks.yml #instance_pools instance_pools === Migration ignores the direct-only resource; it will be created on the next deploy + >>> [CLI] bundle deployment migrate Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json @@ -17,12 +20,14 @@ To undo the migration, remove [TEST_TMP_DIR]/.databricks/bundle/default/resource === After migration: plan shows the direct-only resource as "to add" + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan create instance_pools.pool Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged === Deploy creates it; verify via recorded requests + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created instance_pools.pool @@ -46,5 +51,6 @@ Resources: 1 created, 0 changed, 0 deleted, 1 unchanged } === No further actions planned + >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/migrate/command/basic/output.txt b/acceptance/bundle/migrate/command/basic/output.txt index 0910eb6a274..719f835643f 100644 --- a/acceptance/bundle/migrate/command/basic/output.txt +++ b/acceptance/bundle/migrate/command/basic/output.txt @@ -22,6 +22,7 @@ To undo the migration, remove [TEST_TMP_DIR]/.databricks/bundle/dev/resources.js "s3://[METASTORE_NAME]/metastore/[UUID][0]/volumes/[UUID][1]" === Should show that it's already migrated + >>> musterr [CLI] bundle deployment migrate Error: already using direct engine Details: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=6 lineage="[UUID][2]" @@ -68,6 +69,7 @@ Resources: 0 created, 0 changed, 0 deleted, 3 unchanged "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID][5] interactive/none engine/direct lineage/5MDu0ewFJPg auth/pat" === Should show that it's already migrated + >>> musterr [CLI] bundle deployment migrate Error: already using direct engine Details: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=6 lineage="[UUID][2]" @@ -80,10 +82,12 @@ resources.json: remote direct state serial=6 lineage="[UUID][2]" [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=6 lineage="[UUID][2]" === Extra plan: should have no drift + >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged === Update databricks.yml, run plan & update + >>> update_file.py databricks.yml Migration Migrated >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle plan @@ -102,6 +106,7 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 2 changed, 0 deleted, 1 unchanged === Different target, still on the terraform state from a prior deployment + >>> replay_tfstate.py .databricks/terraform-prod-requests.json .databricks/terraform-prod.tfstate -t prod >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states -t prod diff --git a/acceptance/bundle/migrate/command/dashboards/output.txt b/acceptance/bundle/migrate/command/dashboards/output.txt index 306d160ea38..dc302d87a06 100644 --- a/acceptance/bundle/migrate/command/dashboards/output.txt +++ b/acceptance/bundle/migrate/command/dashboards/output.txt @@ -29,6 +29,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged >>> print_requests.py --get //dashboards === Should show that it's already migrated + >>> musterr [CLI] bundle deployment migrate Error: already using direct engine Details: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=3 lineage="[UUID]" diff --git a/acceptance/bundle/migrate/command/default-python/output.txt b/acceptance/bundle/migrate/command/default-python/output.txt index 8212f47a6c9..f25ab191588 100644 --- a/acceptance/bundle/migrate/command/default-python/output.txt +++ b/acceptance/bundle/migrate/command/default-python/output.txt @@ -167,6 +167,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "resources.jobs.sample_job": { === Extra plan: should have no drift + >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle plan Building python_artifact... update jobs.sample_job diff --git a/acceptance/bundle/migrate/command/grants/output.txt b/acceptance/bundle/migrate/command/grants/output.txt index bd4d8df3fa5..2e1176ced07 100644 --- a/acceptance/bundle/migrate/command/grants/output.txt +++ b/acceptance/bundle/migrate/command/grants/output.txt @@ -27,6 +27,7 @@ Resources: 0 created, 0 changed, 0 deleted, 6 unchanged >>> print_requests.py --get //unity-catalog/permissions === Should show that it's already migrated + >>> musterr [CLI] bundle deployment migrate Error: already using direct engine Details: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=9 lineage="[UUID]" diff --git a/acceptance/bundle/migrate/command/permissions/output.txt b/acceptance/bundle/migrate/command/permissions/output.txt index 5f75d716158..566b6258727 100644 --- a/acceptance/bundle/migrate/command/permissions/output.txt +++ b/acceptance/bundle/migrate/command/permissions/output.txt @@ -53,6 +53,7 @@ Resources: 0 created, 0 changed, 0 deleted, 4 unchanged "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID][2] interactive/none engine/direct lineage/DIsgyJN5V5G auth/pat" === Should show that it's already migrated + >>> musterr [CLI] bundle deployment migrate Error: already using direct engine Details: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=7 lineage="[UUID][3]" diff --git a/acceptance/bundle/open/output.txt b/acceptance/bundle/open/output.txt index 603198f2c19..9501791245a 100644 --- a/acceptance/bundle/open/output.txt +++ b/acceptance/bundle/open/output.txt @@ -1,11 +1,13 @@ === no run key specified + >>> [CLI] bundle open Error: expected a KEY of the resource to open Exit code: 1 === not deployed yet + >>> [CLI] bundle open foo Error: resource does not have a URL associated with it (has it been deployed?) @@ -18,12 +20,15 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Use a fake browser that just prints the URL it would have opened + === open after deployment + >>> [CLI] bundle open foo Opening browser at [DATABRICKS_URL]/jobs/[FOO_ID]?w=[WORKSPACE_ID] [DATABRICKS_URL]/jobs/[FOO_ID]?w=[WORKSPACE_ID] === test auto-completion handler + >>> [CLI] __complete bundle open , foo :4 diff --git a/acceptance/bundle/python/pipeline-development-mutator/output.txt b/acceptance/bundle/python/pipeline-development-mutator/output.txt index ba2610dfe7c..e0c85f202d0 100644 --- a/acceptance/bundle/python/pipeline-development-mutator/output.txt +++ b/acceptance/bundle/python/pipeline-development-mutator/output.txt @@ -1,5 +1,6 @@ === Mutator sets development: false after mode: development stamped it + >>> uv run [UV_ARGS] -q [CLI] bundle validate --output json Warning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes. at resources.pipelines.my_pipeline.development @@ -30,6 +31,7 @@ Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Run sends development: true from mode: development, ignoring the deployed false + >>> uv run [UV_ARGS] -q [CLI] bundle run my_pipeline Warning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes. at resources.pipelines.my_pipeline.development diff --git a/acceptance/bundle/python/restricted-execution/output.txt b/acceptance/bundle/python/restricted-execution/output.txt index 3c9e1b2369a..63ec4fffa33 100644 --- a/acceptance/bundle/python/restricted-execution/output.txt +++ b/acceptance/bundle/python/restricted-execution/output.txt @@ -1,5 +1,6 @@ === Without DATABRICKS_BUNDLE_RESTRICTED_CODE_EXECUTION, all envs are accessible + >>> uv run [UV_ARGS] -q [CLI] bundle summary -o json { "python": { @@ -13,6 +14,7 @@ value_from_env === With DATABRICKS_BUNDLE_RESTRICTED_CODE_EXECUTION=1, no envs are accessible + >>> musterr uv run [UV_ARGS] -q [CLI] bundle summary Error: Running Python code is not allowed when DATABRICKS_BUNDLE_RESTRICTED_CODE_EXECUTION is set diff --git a/acceptance/bundle/python/telemetry/output.txt b/acceptance/bundle/python/telemetry/output.txt index 1b6ba81e813..46dfd6211a9 100644 --- a/acceptance/bundle/python/telemetry/output.txt +++ b/acceptance/bundle/python/telemetry/output.txt @@ -1,5 +1,6 @@ === Deploy with 'python' + >>> uv run [UV_ARGS] -q [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_project/default/files... Created jobs.python_job @@ -36,6 +37,7 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged } === Deploy with 'experimental/python' + >>> update_file.py databricks.yml python: {resources: ["resources:load_resources"], mutators: ["mutators:update_job"]} experimental: {python: {resources: ["resources:load_resources"], mutators: ["mutators:update_job"]}} >>> uv run [UV_ARGS] -q [CLI] bundle deploy diff --git a/acceptance/bundle/quiet-levels/output.txt b/acceptance/bundle/quiet-levels/output.txt index 605cbccfbd8..6639f05b938 100644 --- a/acceptance/bundle/quiet-levels/output.txt +++ b/acceptance/bundle/quiet-levels/output.txt @@ -1,18 +1,22 @@ === plan: actions and summary + >>> [CLI] bundle plan create experiments.my_experiment Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged === plan -q: summary only + >>> [CLI] bundle plan -q Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged === plan -qq: nothing + >>> [CLI] bundle plan -qq === deploy: per-resource lines, progress and summary + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created experiments.my_experiment @@ -22,6 +26,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle destroy --auto-approve -qq === deploy -q: summary only, no per-resource lines + >>> [CLI] bundle deploy -q Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Files: 2 uploaded, 0 deleted @@ -30,9 +35,11 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle destroy --auto-approve -qq === deploy -qq: nothing + >>> [CLI] bundle deploy -qq === destroy: approval list and summary + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.experiments.my_experiment @@ -42,6 +49,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted === destroy -q: same as default, there are no per-resource lines to drop + >>> [CLI] bundle deploy -qq >>> [CLI] bundle destroy --auto-approve -q @@ -53,6 +61,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted === destroy --auto-approve -qq: nothing + >>> [CLI] bundle deploy -qq >>> [CLI] bundle destroy --auto-approve -qq diff --git a/acceptance/bundle/resource_deps/create_error/output.txt b/acceptance/bundle/resource_deps/create_error/output.txt index 9c44fd808d2..3174a6eef98 100644 --- a/acceptance/bundle/resource_deps/create_error/output.txt +++ b/acceptance/bundle/resource_deps/create_error/output.txt @@ -67,6 +67,7 @@ Resources: URL: [DATABRICKS_URL]/jobs/[INDEPENDENT_ID]?w=[WORKSPACE_ID] === Plan should still contain foo and bar + >>> [CLI] bundle plan create jobs.bar create jobs.baz @@ -75,6 +76,7 @@ create jobs.foo Plan: 3 to add, 0 to change, 0 to delete, 1 unchanged === Expecting no difference in the output between first and second deploy + >>> print_requests.py --nostamp //jobs { "method": "POST", diff --git a/acceptance/bundle/resource_deps/escaped_ref/output.txt b/acceptance/bundle/resource_deps/escaped_ref/output.txt index 0bc8c20e1e8..ac3bf2354a3 100644 --- a/acceptance/bundle/resource_deps/escaped_ref/output.txt +++ b/acceptance/bundle/resource_deps/escaped_ref/output.txt @@ -1,5 +1,6 @@ === depends_on: only the real reference creates an edge + >>> jq .plan | map_values(.depends_on) plan.json { "resources.jobs.bar": null, diff --git a/acceptance/bundle/resource_deps/escaped_ref/script b/acceptance/bundle/resource_deps/escaped_ref/script index ef8244e7eeb..8429d5f8481 100644 --- a/acceptance/bundle/resource_deps/escaped_ref/script +++ b/acceptance/bundle/resource_deps/escaped_ref/script @@ -10,7 +10,7 @@ trace jq '.plan | map_values(.depends_on)' plan.json | nostamp # Not traced: the READPLAN variant adds "--plan ...", and the point of that variant is # that the payload below is identical either way. -title "Payload: escaped stays literal, real is resolved\n" +title "Payload: escaped stays literal, real is resolved" $CLI bundle deploy $(readplanarg plan.json) replace_ids.py # --nostamp: under deployment-history recording the job payload carries the DMS deployment diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt index 9e4478fb6c1..6ad29722c43 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt @@ -285,6 +285,7 @@ Resources: 12 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan === Delete all via empty.yml + >>> [CLI] bundle plan >>> [CLI] bundle deploy diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt index ac7b7af1f78..5b0841a5780 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt @@ -285,6 +285,7 @@ Resources: 12 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan === Destroy all via bundle destroy + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.chain_bottom diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/output.txt b/acceptance/bundle/resource_deps/job_id_delete_bar/output.txt index 879c77141ca..181eaea8a00 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/output.txt +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/output.txt @@ -51,6 +51,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Delete bar, keep foo (foo config updated to not depend on bar) + >>> [CLI] bundle plan -o json >>> [CLI] bundle deploy diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/output.txt b/acceptance/bundle/resource_deps/job_id_delete_foo/output.txt index b126ba76227..5447132cbaa 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/output.txt +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/output.txt @@ -51,6 +51,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Delete foo, keep bar + >>> [CLI] bundle plan -o json >>> [CLI] bundle deploy diff --git a/acceptance/bundle/resource_deps/jobs_update/output.txt b/acceptance/bundle/resource_deps/jobs_update/output.txt index 47f5b942227..a4dd22c3d4e 100644 --- a/acceptance/bundle/resource_deps/jobs_update/output.txt +++ b/acceptance/bundle/resource_deps/jobs_update/output.txt @@ -18,6 +18,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === Update trigger.periodic.unit and re-deploy; jobs.bar is unchanged + >>> update_file.py databricks.yml DAYS HOURS >>> [CLI] bundle plan @@ -37,6 +38,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === Fetch job ID and verify remote state + >>> [CLI] jobs get [FOO_ID] >>> [CLI] jobs get [BAR_ID] diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/output.txt b/acceptance/bundle/resource_deps/jobs_update_remote/output.txt index c0df8556b7a..6584341f5f9 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/output.txt +++ b/acceptance/bundle/resource_deps/jobs_update_remote/output.txt @@ -63,6 +63,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === Update trigger.periodic.unit remotely and re-deploy; jobs.bar is unchanged + >>> envsubst >>> [CLI] jobs reset --json @job_update.json diff --git a/acceptance/bundle/resource_deps/loop_jobs/output.txt b/acceptance/bundle/resource_deps/loop_jobs/output.txt index 8ca2facea70..1d157cb8352 100644 --- a/acceptance/bundle/resource_deps/loop_jobs/output.txt +++ b/acceptance/bundle/resource_deps/loop_jobs/output.txt @@ -1,5 +1,6 @@ === Badness: validate could detect this and fail as well + >>> [CLI] bundle validate Name: test-bundle Target: default diff --git a/acceptance/bundle/resource_deps/loop_self/output.txt b/acceptance/bundle/resource_deps/loop_self/output.txt index 096498c3d00..7fddf0690a2 100644 --- a/acceptance/bundle/resource_deps/loop_self/output.txt +++ b/acceptance/bundle/resource_deps/loop_self/output.txt @@ -1,5 +1,6 @@ -=== Badness: validate could detect this and fail as wellName: test-bundle +=== Badness: validate could detect this and fail as well +Name: test-bundle Target: default Workspace: User: [USERNAME] diff --git a/acceptance/bundle/resource_deps/model_id_ref/output.txt b/acceptance/bundle/resource_deps/model_id_ref/output.txt index ccbb4932369..5a9acaa1224 100644 --- a/acceptance/bundle/resource_deps/model_id_ref/output.txt +++ b/acceptance/bundle/resource_deps/model_id_ref/output.txt @@ -1,5 +1,6 @@ === the consumer job implicitly depends on the model + >>> [CLI] bundle plan create jobs.consumer create models.my_model @@ -7,6 +8,7 @@ create models.my_model Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged === after deploy, registered_model_id is resolved to the model's numeric id + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created jobs.consumer diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt index 4ab49471753..7c0774b248c 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt +++ b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt @@ -18,6 +18,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === Update storage, triggering recreate for pipeline; this means updating downstream deps + >>> update_file.py databricks.yml storage: dbfs:/my-storage storage: dbfs:/my-new-storage >>> [CLI] bundle plan @@ -42,6 +43,7 @@ Resources: 1 created, 1 changed, 1 deleted, 0 unchanged >>> print_requests.py --nostamp //jobs //pipelines ^//api/2.0/bundle === Fetch resource IDs and verify remote state + >>> musterr [CLI] pipelines get [FOO_ID] Error: The specified pipeline [FOO_ID] was not found. @@ -93,6 +95,7 @@ Error: The specified pipeline [FOO_ID] was not found. } === Follow up plan & deploy do nothing + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/resource_deps/unicode_reference/output.txt b/acceptance/bundle/resource_deps/unicode_reference/output.txt index 969eade1b86..27a5bc23aa6 100644 --- a/acceptance/bundle/resource_deps/unicode_reference/output.txt +++ b/acceptance/bundle/resource_deps/unicode_reference/output.txt @@ -1,5 +1,6 @@ === Unicode job names resolve in resources references + >>> [CLI] bundle validate Name: test-bundle Target: default diff --git a/acceptance/bundle/resources/alerts/basic/output.txt b/acceptance/bundle/resources/alerts/basic/output.txt index 3944c2a21ff..a086fb92563 100644 --- a/acceptance/bundle/resources/alerts/basic/output.txt +++ b/acceptance/bundle/resources/alerts/basic/output.txt @@ -38,6 +38,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === assert that permissions are applied + >>> [CLI] permissions get alertsv2 [ALERT_ID] { "user_name": "deco-test-user@databricks.com", @@ -50,6 +51,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === assert that no permanent drift happens + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/alerts/with_file/output.txt b/acceptance/bundle/resources/alerts/with_file/output.txt index 637a6291c19..fa4ad5b9555 100644 --- a/acceptance/bundle/resources/alerts/with_file/output.txt +++ b/acceptance/bundle/resources/alerts/with_file/output.txt @@ -68,6 +68,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === assert that the uploaded alert file is the same as the local one + >>> [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/alerts-with-file-[UNIQUE_NAME]/default/resources/myalert.dbalert.json >>> diff local_normalized.json exported_normalized.json diff --git a/acceptance/bundle/resources/alerts/with_file_run_from_subdir/output.txt b/acceptance/bundle/resources/alerts/with_file_run_from_subdir/output.txt index af27a587c49..c814a34a196 100644 --- a/acceptance/bundle/resources/alerts/with_file_run_from_subdir/output.txt +++ b/acceptance/bundle/resources/alerts/with_file_run_from_subdir/output.txt @@ -1,5 +1,6 @@ === validate from the bundle root + >>> [CLI] bundle validate -o json { "custom_summary": "My alert from file", @@ -33,6 +34,7 @@ } === validate from a subdirectory of the bundle + >>> [CLI] bundle validate -o json { "custom_summary": "My alert from file", diff --git a/acceptance/bundle/resources/apps/config-drift-stopped/output.txt b/acceptance/bundle/resources/apps/config-drift-stopped/output.txt index 50c91532377..a0f3123d7c0 100644 --- a/acceptance/bundle/resources/apps/config-drift-stopped/output.txt +++ b/acceptance/bundle/resources/apps/config-drift-stopped/output.txt @@ -1,5 +1,6 @@ === Deploy with started=true so the app has an active deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/config-drift-stopped-[UNIQUE_NAME]/default/files... ✓ Deployment succeeded @@ -8,6 +9,7 @@ Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Change config while running: drift is detected (active deployment present) + >>> update_file.py databricks.yml original_value changed_value >>> [CLI] bundle plan -o json @@ -20,10 +22,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Stop the app: the backend clears the active deployment + >>> [CLI] apps stop [UNIQUE_NAME] "STOPPED" === Same config change is now skipped: no active deployment to compare against + >>> [CLI] bundle plan -o json { "config": { diff --git a/acceptance/bundle/resources/apps/config-drift/output.txt b/acceptance/bundle/resources/apps/config-drift/output.txt index e12c69659fc..332dfba64a9 100644 --- a/acceptance/bundle/resources/apps/config-drift/output.txt +++ b/acceptance/bundle/resources/apps/config-drift/output.txt @@ -6,18 +6,22 @@ Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Verify no drift after deploy + >>> [CLI] bundle plan -o json >>> [CLI] apps get [UNIQUE_NAME] --output json === Simulate out-of-band deployment with changed command and env + === Plan should detect config drift + >>> [CLI] bundle plan update apps.myapp Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged === Redeploy to fix drift + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/config-drift-[UNIQUE_NAME]/default/files... Updated apps.myapp @@ -25,6 +29,7 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Verify no drift after fix + >>> [CLI] bundle plan -o json >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/apps/config-env-no-drift/output.txt b/acceptance/bundle/resources/apps/config-env-no-drift/output.txt index 82d16c28a37..51b5ab4b7a7 100644 --- a/acceptance/bundle/resources/apps/config-env-no-drift/output.txt +++ b/acceptance/bundle/resources/apps/config-env-no-drift/output.txt @@ -1,5 +1,6 @@ === Deploy a started app with config.env entries set via value and value_from + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/apps-config-env-no-drift/default/files... ✓ Deployment succeeded @@ -8,6 +9,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === The Apps API returns deployment env vars with their name only + >>> [CLI] bundle plan -o json [ { @@ -19,6 +21,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === Re-plan is clean: value and value_from are input-only and must not drift + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -39,6 +42,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } === A local change to an env value still plans an update + >>> update_file.py databricks.yml value: hello value: goodbye >>> [CLI] bundle plan @@ -62,6 +66,7 @@ Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged } === Deploy applies the change and the plan is clean again + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/apps-config-env-no-drift/default/files... ✓ Deployment succeeded diff --git a/acceptance/bundle/resources/apps/config-no-deployment/output.txt b/acceptance/bundle/resources/apps/config-no-deployment/output.txt index 4887cf920df..edfde35bf49 100644 --- a/acceptance/bundle/resources/apps/config-no-deployment/output.txt +++ b/acceptance/bundle/resources/apps/config-no-deployment/output.txt @@ -1,5 +1,6 @@ === Deploy: app created with no_compute; config is not deployed until the app starts + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/app-config-no-deployment/default/files... Created apps.myapp @@ -7,10 +8,12 @@ Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Plan is a no-op: config/source_code_path drift is skipped while the app has no active deployment + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Both deploy-only fields are skipped with reason "no active deployment" + >>> [CLI] bundle plan -o json { "config": { diff --git a/acceptance/bundle/resources/apps/git-source-no-deployment/output.txt b/acceptance/bundle/resources/apps/git-source-no-deployment/output.txt index 70a931d7fb0..84def7b23cb 100644 --- a/acceptance/bundle/resources/apps/git-source-no-deployment/output.txt +++ b/acceptance/bundle/resources/apps/git-source-no-deployment/output.txt @@ -1,5 +1,6 @@ === Deploy: app created with no_compute; git_source is not deployed until the app starts + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/app-git-source-no-deployment/default/files... Created apps.myapp @@ -7,10 +8,12 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Plan is a no-op: git_source drift is skipped while the app has no active deployment + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === git_source is skipped with reason "no active deployment" + >>> [CLI] bundle plan -o json { "git_source": { diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/output.txt b/acceptance/bundle/resources/apps/lifecycle-started-omitted/output.txt index 701f2a7743b..3202e7480a8 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-omitted/output.txt +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/output.txt @@ -1,5 +1,6 @@ === (started omitted) -> deploy: app created with no_compute=true, no start/stop requests + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-omitted-[UNIQUE_NAME]/default/files... Created apps.mykey @@ -20,6 +21,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === (started omitted) -> started: false -> deploy: no start/stop requests + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-omitted-[UNIQUE_NAME]/default/files... Files: 1 uploaded, 0 deleted @@ -28,6 +30,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged >>> print_app_requests === started: false -> (started omitted) -> deploy: no start/stop requests + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-omitted-[UNIQUE_NAME]/default/files... Files: 1 uploaded, 0 deleted @@ -36,6 +39,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged >>> print_app_requests === (started omitted) -> started: true -> deploy: app started + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-omitted-[UNIQUE_NAME]/default/files... Updated apps.mykey @@ -58,6 +62,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === started: true -> (started omitted) -> deploy: no start/stop requests (compute stays as-is) + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-omitted-[UNIQUE_NAME]/default/files... Updated apps.mykey @@ -66,7 +71,8 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> print_app_requests -=== (started omitted, app running) -> bundle plan shows no driftPlan: no drift detected +=== (started omitted, app running) -> bundle plan shows no drift +Plan: no drift detected >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt b/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt index b6631453d16..0e88990ab01 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt @@ -1,5 +1,6 @@ === Deploy with started=false: app created without compute (no_compute=true) + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default/files... Created apps.mykey @@ -23,6 +24,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "STOPPED" === Toggle started=false -> started=true: only Start should be called, no Update + >>> update_file.py databricks.yml started: false started: true >>> [CLI] bundle deploy @@ -50,6 +52,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "ACTIVE" === Toggle started=true -> started=false: only Stop should be called, no Update + >>> update_file.py databricks.yml started: true started: false >>> [CLI] bundle deploy diff --git a/acceptance/bundle/resources/apps/lifecycle-started/output.txt b/acceptance/bundle/resources/apps/lifecycle-started/output.txt index becd0f95b05..ace0ac41a61 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/output.txt +++ b/acceptance/bundle/resources/apps/lifecycle-started/output.txt @@ -1,5 +1,6 @@ === Deploy with started=true: app created with compute running + >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... Created apps.myapp @@ -28,6 +29,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "ACTIVE" === Re-deploy with description change: code deployed again + >>> update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION >>> errcode [CLI] bundle deploy @@ -47,6 +49,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Stop app externally while config says started=true: plan detects drift + >>> errcode [CLI] apps stop [UNIQUE_NAME] "STOPPED" @@ -56,6 +59,7 @@ update apps.myapp Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged === Deploy fixes the drift: compute restarted + >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... Updated apps.myapp @@ -63,6 +67,7 @@ Files: 0 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Stop app externally, then deploy with started=false: app stays stopped + >>> errcode [CLI] apps stop [UNIQUE_NAME] "STOPPED" @@ -98,6 +103,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "STOPPED" === Deploy with started=true: compute restarted and code deployed + >>> update_file.py databricks.yml started: false started: true >>> update_file.py databricks.yml MY_APP_DESCRIPTION_2 MY_APP_DESCRIPTION_3 diff --git a/acceptance/bundle/resources/apps/readplan-lifecycle/output.txt b/acceptance/bundle/resources/apps/readplan-lifecycle/output.txt index fc9f8041709..55d7e1e331a 100644 --- a/acceptance/bundle/resources/apps/readplan-lifecycle/output.txt +++ b/acceptance/bundle/resources/apps/readplan-lifecycle/output.txt @@ -1,5 +1,6 @@ === Initial deploy: app created and started + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created apps.myapp @@ -9,6 +10,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //apps/myapp/start === Change description and deploy from plan: no extra start + >>> update_file.py databricks.yml description: initial description: updated Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Updated apps.myapp diff --git a/acceptance/bundle/resources/apps/uc-securable-drift/output.txt b/acceptance/bundle/resources/apps/uc-securable-drift/output.txt index 2f4713ea9bf..bb751516f8e 100644 --- a/acceptance/bundle/resources/apps/uc-securable-drift/output.txt +++ b/acceptance/bundle/resources/apps/uc-securable-drift/output.txt @@ -1,5 +1,6 @@ === Deploy an app with UC TABLE and VOLUME securables + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/apps-uc-securable-drift/default/files... Created apps.myapp @@ -7,10 +8,12 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Re-plan is clean: the TABLE securable's server-computed securable_kind is output-only and must not drift + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === securable_kind is classified output-only (skip); the VOLUME securable has none + >>> [CLI] bundle plan -o json { "resources[0].uc_securable.securable_kind": { diff --git a/acceptance/bundle/resources/apps/update-compute-size/output.txt b/acceptance/bundle/resources/apps/update-compute-size/output.txt index be44bf5f2e7..3434c24e629 100644 --- a/acceptance/bundle/resources/apps/update-compute-size/output.txt +++ b/acceptance/bundle/resources/apps/update-compute-size/output.txt @@ -1,5 +1,6 @@ === Deploy with compute_size declared + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created apps.mykey @@ -9,6 +10,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests === Update description while compute_size is still declared + >>> update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION >>> [CLI] bundle deploy @@ -20,6 +22,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> print_requests === Remove compute_size and update description + >>> update_file.py databricks.yml MY_APP_DESCRIPTION MY_APP_DESCRIPTION_2 >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/apps/update/output.txt b/acceptance/bundle/resources/apps/update/output.txt index d0d9790878e..609fdb5c409 100644 --- a/acceptance/bundle/resources/apps/update/output.txt +++ b/acceptance/bundle/resources/apps/update/output.txt @@ -25,6 +25,7 @@ Resources: URL: (not deployed) === Update description and re-deploy + >>> update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION >>> [CLI] bundle plan @@ -53,6 +54,7 @@ Resources: URL: (not deployed) === Update name and re-deploy + >>> update_file.py databricks.yml myappname mynewappname >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/catalogs/auto-approve/output.txt b/acceptance/bundle/resources/catalogs/auto-approve/output.txt index 74d1a53ab73..91410c28e8d 100644 --- a/acceptance/bundle/resources/catalogs/auto-approve/output.txt +++ b/acceptance/bundle/resources/catalogs/auto-approve/output.txt @@ -7,6 +7,7 @@ Files: 4 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Assert the catalog is created + >>> [CLI] catalogs get test-catalog-[UNIQUE_NAME] { "full_name": "test-catalog-[UNIQUE_NAME]", @@ -14,6 +15,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Assert the schema is created and uses the catalog + >>> [CLI] schemas get test-catalog-[UNIQUE_NAME].test-schema-[UNIQUE_NAME] { "full_name": "test-catalog-[UNIQUE_NAME].test-schema-[UNIQUE_NAME]", @@ -24,6 +26,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Create a volume in the schema, and add a file to it. This ensures that the catalog has some data in it and deletion will fail unless force deletion is enabled. + >>> [CLI] volumes create test-catalog-[UNIQUE_NAME] test-schema-[UNIQUE_NAME] test-volume-[UNIQUE_NAME] MANAGED { "full_name": "test-catalog-[UNIQUE_NAME].test-schema-[UNIQUE_NAME].test-volume-[UNIQUE_NAME]" @@ -33,9 +36,11 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged test-file-[UNIQUE_NAME].txt -> dbfs:/Volumes/test-catalog-[UNIQUE_NAME]/test-schema-[UNIQUE_NAME]/test-volume-[UNIQUE_NAME]/test-file-[UNIQUE_NAME].txt === Remove the UC catalog from the resource configuration. + >>> rm catalog.yml === Try to redeploy the bundle - should fail without --auto-approve + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Error: cannot plan resources.schemas.foo: cannot resolve "${resources.catalogs.bar.name}": internal error: resources.catalogs.bar: action is "delete" missing new_state @@ -45,6 +50,7 @@ Error: planning failed Files: 1 uploaded, 1 deleted === Test cleanup + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.catalogs.bar @@ -58,4 +64,5 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 2 deleted === Assert the catalog is deleted + Exit code: 1 diff --git a/acceptance/bundle/resources/catalogs/basic/output.txt b/acceptance/bundle/resources/catalogs/basic/output.txt index fdc31856ca9..5b90d0c0b8f 100644 --- a/acceptance/bundle/resources/catalogs/basic/output.txt +++ b/acceptance/bundle/resources/catalogs/basic/output.txt @@ -1,5 +1,6 @@ === Deploy bundle with catalog + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created catalogs.test_catalog @@ -7,6 +8,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Assert the catalog is created + >>> [CLI] catalogs get test_catalog_[UNIQUE_NAME] { "name": "test_catalog_[UNIQUE_NAME]", @@ -17,7 +19,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Update catalog comment + === Redeploy with updated comment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Updated catalogs.test_catalog @@ -25,6 +29,7 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Assert the catalog comment is updated + >>> [CLI] catalogs get test_catalog_[UNIQUE_NAME] { "name": "test_catalog_[UNIQUE_NAME]", @@ -32,6 +37,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Test cleanup + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.catalogs.test_catalog @@ -40,4 +46,4 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted -=== Assert the catalog is deleted \ No newline at end of file +=== Assert the catalog is deleted diff --git a/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt index 58bebd5aac2..65bbc43d3df 100644 --- a/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt @@ -1,5 +1,6 @@ === Deploy a catalog that does not declare a comment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created catalogs.catalog1 @@ -7,7 +8,9 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Set the comment out of band, the way Catalog Explorer does + === The remote comment is drift, so the plan updates the catalog + >>> [CLI] bundle plan --output json { "comment": { @@ -17,6 +20,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Redeploy clears the comment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated catalogs.catalog1 @@ -24,16 +28,19 @@ Files: 0 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === The comment is gone + >>> [CLI] api get /api/2.1/unity-catalog/catalogs/test-catalog-[UNIQUE_NAME] "" === Redeploy is a no-op + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged === The UpdateCatalog payloads the deploys sent + >>> print_requests.py --method PATCH //unity-catalog/catalogs { "method": "PATCH", diff --git a/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt b/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt index 5c22507bb39..2f01b81cc37 100644 --- a/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt +++ b/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created catalogs.catalog1 @@ -7,10 +8,12 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Plan is a no-op despite UC auto-populating managed properties + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The remote-only properties map is skipped as a backend default (confirms the matched rule) + >>> [CLI] bundle plan --output json { "properties": { @@ -29,6 +32,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } === Redeploy is a no-op (no UpdateCatalog call) + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 0 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/catalogs/with-schemas/output.txt b/acceptance/bundle/resources/catalogs/with-schemas/output.txt index 07928270eb8..209785b8189 100644 --- a/acceptance/bundle/resources/catalogs/with-schemas/output.txt +++ b/acceptance/bundle/resources/catalogs/with-schemas/output.txt @@ -1,5 +1,6 @@ === Deploy bundle with catalog and schema + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created catalogs.test_catalog @@ -8,6 +9,7 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Assert the catalog is created + >>> [CLI] catalogs get test_catalog_[UNIQUE_NAME] { "name": "test_catalog_[UNIQUE_NAME]", @@ -18,6 +20,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Assert schema is created in the custom catalog + >>> [CLI] schemas get test_catalog_[UNIQUE_NAME].schema1 { "full_name": "test_catalog_[UNIQUE_NAME].schema1", @@ -26,8 +29,11 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Verify schema belongs to the test catalog + === Update catalog comment + === Redeploy with updated catalog + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Updated catalogs.test_catalog @@ -35,6 +41,7 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged === Assert catalog comment is updated + >>> [CLI] catalogs get test_catalog_[UNIQUE_NAME] { "name": "test_catalog_[UNIQUE_NAME]", @@ -42,12 +49,14 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } === Assert schema still exists after catalog update + >>> [CLI] schemas get test_catalog_[UNIQUE_NAME].schema1 { "full_name": "test_catalog_[UNIQUE_NAME].schema1" } === Test cleanup + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.catalogs.test_catalog @@ -61,4 +70,5 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 2 deleted === Assert schema is deleted -=== Assert the catalog is deleted \ No newline at end of file + +=== Assert the catalog is deleted diff --git a/acceptance/bundle/resources/cluster_policies/backend_normalization/output.txt b/acceptance/bundle/resources/cluster_policies/backend_normalization/output.txt index 61848083f68..e1eeb76fae0 100644 --- a/acceptance/bundle/resources/cluster_policies/backend_normalization/output.txt +++ b/acceptance/bundle/resources/cluster_policies/backend_normalization/output.txt @@ -8,10 +8,12 @@ Files: 2 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === [json_string] definition returned by the backend (GET) + >>> [CLI] cluster-policies get [JSON_STRING_ID] -o json {"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}} === [multiline] definition returned by the backend (GET) + >>> [CLI] cluster-policies get [MULTILINE_ID] -o json { "spark_version": { @@ -21,10 +23,12 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } === [yaml] definition returned by the backend (GET) + >>> [CLI] cluster-policies get [YAML_ID] -o json {"autotermination_minutes":{"maxValue":120,"type":"range"},"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}} === Plan after deploy: any drift means the backend reformatted a definition + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged diff --git a/acceptance/bundle/resources/cluster_policies/basic/output.txt b/acceptance/bundle/resources/cluster_policies/basic/output.txt index 0ba76cc3b55..234b153f5c0 100644 --- a/acceptance/bundle/resources/cluster_policies/basic/output.txt +++ b/acceptance/bundle/resources/cluster_policies/basic/output.txt @@ -35,6 +35,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Verify the create request + >>> print_requests.py //policies/clusters/create { "method": "POST", @@ -58,6 +59,7 @@ Resources: URL: [DATABRICKS_URL]/compute/policies/[TEST_CLUSTER_POLICY_ID]?w=[WORKSPACE_ID] === Update the cluster policy name + >>> update_file.py databricks.yml my_cluster_policy my_cluster_policy_2 >>> [CLI] bundle deploy @@ -67,6 +69,7 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Verify the update request + >>> print_requests.py //policies/clusters/edit { "method": "POST", @@ -91,6 +94,7 @@ Resources: URL: [DATABRICKS_URL]/compute/policies/[TEST_CLUSTER_POLICY_ID]?w=[WORKSPACE_ID] === Destroy the cluster policy + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.cluster_policies.test_cluster_policy @@ -100,6 +104,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted === Verify the destroy request + >>> print_requests.py //policies/clusters/delete { "method": "POST", diff --git a/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/script b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/script index 4d83ab4b716..8ae30c4b72f 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/script +++ b/acceptance/bundle/resources/cluster_policies/definition_and_family_conflict/script @@ -5,5 +5,5 @@ cleanup() { } trap cleanup EXIT -title "Deploying a policy with both definition and policy_family_id must fail\n" +title "Deploying a policy with both definition and policy_family_id must fail" musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt b/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt index 321e18516f2..fdd116d4e99 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt +++ b/acceptance/bundle/resources/cluster_policies/definition_multiline/output.txt @@ -14,6 +14,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Create body preserves the block-scalar definition as a newline-escaped string + >>> print_requests.py //policies/clusters { "method": "POST", diff --git a/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt b/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt index 68e3de9cffd..16b25898d89 100644 --- a/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt +++ b/acceptance/bundle/resources/cluster_policies/definition_yaml/output.txt @@ -14,6 +14,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Native YAML definition serializes to compact JSON, preserving numbers and booleans as JSON types (not quoted strings) + >>> print_requests.py //policies/clusters { "method": "POST", diff --git a/acceptance/bundle/resources/cluster_policies/job_ref/output.txt b/acceptance/bundle/resources/cluster_policies/job_ref/output.txt index 1172c061bde..278a4709786 100644 --- a/acceptance/bundle/resources/cluster_policies/job_ref/output.txt +++ b/acceptance/bundle/resources/cluster_policies/job_ref/output.txt @@ -7,6 +7,7 @@ Files: 1 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Deploy requests in dependency order: policy create precedes job create, job carries resolved policy id + >>> print_requests.py //policies/clusters //jobs --nostamp { "method": "POST", @@ -45,6 +46,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Update both the policy and the job + >>> update_file.py databricks.yml my_policy my_policy_2 >>> update_file.py databricks.yml my_job my_job_2 @@ -98,6 +100,7 @@ Resources: 0 created, 2 changed, 0 deleted, 0 unchanged } === Destroy: job delete precedes policy delete + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.cluster_policies.pol diff --git a/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt b/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt index d476def4bd3..24aa5f88aeb 100644 --- a/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt +++ b/acceptance/bundle/resources/cluster_policies/out_of_band_change/output.txt @@ -6,19 +6,23 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Plan is a no-op immediately after deploy + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Edit the policy definition out of band + >>> [CLI] cluster-policies edit --json {"policy_id":"[TEST_CLUSTER_POLICY_ID]","name":"my_cluster_policy_oob-[UNIQUE_NAME]","definition":"{\"spark_version\":{\"type\":\"fixed\",\"value\":\"14.3.x-scala2.12\"}}"} === Plan detects the drift + >>> [CLI] bundle plan update cluster_policies.test_cluster_policy Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged === Redeploy reconciles the policy back to the configured definition + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Updated cluster_policies.test_cluster_policy @@ -26,6 +30,7 @@ Files: 0 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Verify the edit request restored the configured definition + >>> print_requests.py //policies/clusters/edit { "method": "POST", @@ -38,6 +43,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Plan is a no-op again + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/cluster_policies/permissions/levels/output.txt b/acceptance/bundle/resources/cluster_policies/permissions/levels/output.txt index 37d4c20606e..e228fd2b3ba 100644 --- a/acceptance/bundle/resources/cluster_policies/permissions/levels/output.txt +++ b/acceptance/bundle/resources/cluster_policies/permissions/levels/output.txt @@ -21,6 +21,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === CAN_MANAGE is rejected + >>> [CLI] bundle destroy --auto-approve Warning: invalid value "CAN_MANAGE" for enum field. Valid values are [CAN_USE] at resources.cluster_policies.test_cluster_policy.permissions[0].level diff --git a/acceptance/bundle/resources/cluster_policies/permissions/out_of_band_deletion/output.txt b/acceptance/bundle/resources/cluster_policies/permissions/out_of_band_deletion/output.txt index e64e6628d1b..777a6c3a030 100644 --- a/acceptance/bundle/resources/cluster_policies/permissions/out_of_band_deletion/output.txt +++ b/acceptance/bundle/resources/cluster_policies/permissions/out_of_band_deletion/output.txt @@ -7,6 +7,7 @@ Files: 1 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Permissions are set on the live policy + >>> [CLI] api get /api/2.0/permissions/cluster-policies/[TEST_CLUSTER_POLICY_ID] { "object_type": "cluster-policy", @@ -24,9 +25,11 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Delete the policy out of band + >>> [CLI] cluster-policies delete [TEST_CLUSTER_POLICY_ID] === Permissions API still returns the ACLs for the deleted policy (no 404) + >>> [CLI] api get /api/2.0/permissions/cluster-policies/[TEST_CLUSTER_POLICY_ID] { "object_type": "cluster-policy", @@ -44,6 +47,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Plan recreates the gone policy + >>> [CLI] bundle plan create cluster_policies.test_cluster_policy update cluster_policies.test_cluster_policy.permissions diff --git a/acceptance/bundle/resources/cluster_policies/permissions/out_of_band_grant/output.txt b/acceptance/bundle/resources/cluster_policies/permissions/out_of_band_grant/output.txt index 886ab1d7f1d..ef361efa9bc 100644 --- a/acceptance/bundle/resources/cluster_policies/permissions/out_of_band_grant/output.txt +++ b/acceptance/bundle/resources/cluster_policies/permissions/out_of_band_grant/output.txt @@ -7,10 +7,12 @@ Files: 1 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Plan is a no-op immediately after deploy + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === Grant an extra principal out of band + >>> [CLI] cluster-policies set-permissions [TEST_CLUSTER_POLICY_ID] --json {"access_control_list":[{"group_name":"users","permission_level":"CAN_USE"},{"user_name":"intruder@example.com","permission_level":"CAN_USE"}]} { "access_control_list": [ @@ -39,12 +41,14 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged } === Plan detects the permission drift + >>> [CLI] bundle plan update cluster_policies.test_cluster_policy.permissions Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged === Redeploy reconciles the ACL back to the configured grants + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test_cluster_policy_permissions_drift/default/files... Updated cluster_policies.test_cluster_policy.permissions @@ -66,6 +70,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } === Plan is a no-op again + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_definition/output.txt b/acceptance/bundle/resources/cluster_policies/policy_family_definition/output.txt index 8389a962f28..c187259ae4a 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_family_definition/output.txt +++ b/acceptance/bundle/resources/cluster_policies/policy_family_definition/output.txt @@ -6,10 +6,12 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Plan is a no-op after deploy (server-computed definition is not drift) + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Redeploy issues no edit + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Files: 0 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/output.txt b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/output.txt index 502906cf07d..07e2025c94b 100644 --- a/acceptance/bundle/resources/cluster_policies/policy_family_overrides/output.txt +++ b/acceptance/bundle/resources/cluster_policies/policy_family_overrides/output.txt @@ -27,6 +27,7 @@ Files: 2 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Create bodies carry policy_family_id and overrides as a JSON string (YAML normalized, JSON preserved) + >>> print_requests.py //policies/clusters/create --sort { "method": "POST", diff --git a/acceptance/bundle/resources/clusters/cluster_policy/script b/acceptance/bundle/resources/clusters/cluster_policy/script index d32f03284ae..4169fff3fcf 100644 --- a/acceptance/bundle/resources/clusters/cluster_policy/script +++ b/acceptance/bundle/resources/clusters/cluster_policy/script @@ -7,9 +7,9 @@ trap cleanup EXIT trace $CLI bundle deploy -title "A cluster policy tag on a standalone cluster is not drift\n" +title "A cluster policy tag on a standalone cluster is not drift" trace $CLI bundle plan -title "...classified remote_addition\n" +title "...classified remote_addition" trace $CLI bundle plan -o json | jq -S '.plan["resources.clusters.my_cluster"].changes | with_entries(select(.key | test("custom_tags")))' diff --git a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/script b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/script index 7455e441da8..fd32c88178c 100644 --- a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/script +++ b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/script @@ -9,6 +9,6 @@ trap cleanup EXIT trace $CLI bundle deploy print_requests.py //cluster > out.requests.json -title "Cluster should exist after bundle deployment:\n" +title "Cluster should exist after bundle deployment:" CLUSTER_ID=`read_id.py test_cluster` $CLI clusters get "${CLUSTER_ID}" | gron.py | grep json.gcp_attributes > out.result.txt diff --git a/acceptance/bundle/resources/clusters/deploy/local_ssd_count_update/script b/acceptance/bundle/resources/clusters/deploy/local_ssd_count_update/script index 4ce07d742fc..d9e8266eeaf 100644 --- a/acceptance/bundle/resources/clusters/deploy/local_ssd_count_update/script +++ b/acceptance/bundle/resources/clusters/deploy/local_ssd_count_update/script @@ -4,34 +4,34 @@ cleanup() { } trap cleanup EXIT -title "Deploy 1: create the cluster with gcp_attributes but no local_ssd_count\n" +title "Deploy 1: create the cluster with gcp_attributes but no local_ssd_count" trace $CLI bundle deploy -qq -title "gcp_attributes sent on create (local_ssd_count absent):\n" +title "gcp_attributes sent on create (local_ssd_count absent):" print_requests.py //clusters/create | jq '.body.gcp_attributes' -title "Now the user adds an explicit local_ssd_count: 0 to the config\n" +title "Now the user adds an explicit local_ssd_count: 0 to the config" update_file.py databricks.yml "#local_ssd_count: 0" "local_ssd_count: 0" -title "Deploy 2: bundle should detect and apply the local_ssd_count change\n" +title "Deploy 2: bundle should detect and apply the local_ssd_count change" trace $CLI bundle plan -title "local_ssd_count change in the plan:\n" +title "local_ssd_count change in the plan:" $CLI bundle plan -o json | jq '.plan[].changes["gcp_attributes.local_ssd_count"]' trace $CLI bundle deploy -qq -title "gcp_attributes sent on update (empty means the change was NOT applied):\n" +title "gcp_attributes sent on update (empty means the change was NOT applied):" print_requests.py //clusters/edit | jq '.body.gcp_attributes' -title "Deploy 3: no config change, should converge to a no-op\n" +title "Deploy 3: no config change, should converge to a no-op" trace $CLI bundle plan -title "Now the user removes local_ssd_count again\n" +title "Now the user removes local_ssd_count again" update_file.py databricks.yml "local_ssd_count: 0" "#local_ssd_count: 0" -title "Deploy 4: removing the explicit 0 is a change, symmetric with setting it\n" +title "Deploy 4: removing the explicit 0 is a change, symmetric with setting it" trace $CLI bundle plan -title "local_ssd_count change in the plan:\n" +title "local_ssd_count change in the plan:" $CLI bundle plan -o json | jq '.plan[].changes["gcp_attributes.local_ssd_count"]' trace $CLI bundle deploy -qq -title "Deploy 5: no config change, should converge to a no-op\n" +title "Deploy 5: no config change, should converge to a no-op" trace $CLI bundle plan diff --git a/acceptance/bundle/resources/clusters/deploy/simple/script b/acceptance/bundle/resources/clusters/deploy/simple/script index 12977cedb09..12b3be5c8fa 100644 --- a/acceptance/bundle/resources/clusters/deploy/simple/script +++ b/acceptance/bundle/resources/clusters/deploy/simple/script @@ -7,6 +7,6 @@ trap cleanup EXIT trace $CLI bundle deploy -title "Cluster should exist after bundle deployment:\n" +title "Cluster should exist after bundle deployment:" CLUSTER_ID=$($CLI bundle summary -o json | jq -r '.resources.clusters.test_cluster.id') $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers}' diff --git a/acceptance/bundle/resources/clusters/deploy/spot_bid_max_price_update/script b/acceptance/bundle/resources/clusters/deploy/spot_bid_max_price_update/script index 450f4e695fd..cb8513653f4 100644 --- a/acceptance/bundle/resources/clusters/deploy/spot_bid_max_price_update/script +++ b/acceptance/bundle/resources/clusters/deploy/spot_bid_max_price_update/script @@ -4,34 +4,34 @@ cleanup() { } trap cleanup EXIT -title "Deploy 1: create the cluster with azure_attributes but no spot_bid_max_price\n" +title "Deploy 1: create the cluster with azure_attributes but no spot_bid_max_price" trace $CLI bundle deploy -qq -title "azure_attributes sent on create (spot_bid_max_price absent):\n" +title "azure_attributes sent on create (spot_bid_max_price absent):" print_requests.py //clusters/create | jq '.body.azure_attributes' -title "Now the user adds an explicit spot_bid_max_price: 0 to the config\n" +title "Now the user adds an explicit spot_bid_max_price: 0 to the config" update_file.py databricks.yml "#spot_bid_max_price: 0" "spot_bid_max_price: 0" -title "Deploy 2: bundle should detect and apply the spot_bid_max_price change\n" +title "Deploy 2: bundle should detect and apply the spot_bid_max_price change" trace $CLI bundle plan -title "spot_bid_max_price change in the plan:\n" +title "spot_bid_max_price change in the plan:" $CLI bundle plan -o json | jq '.plan[].changes["azure_attributes.spot_bid_max_price"]' trace $CLI bundle deploy -qq -title "azure_attributes sent on update (empty means the change was NOT applied):\n" +title "azure_attributes sent on update (empty means the change was NOT applied):" print_requests.py //clusters/edit | jq '.body.azure_attributes' -title "Deploy 3: no config change, should converge to a no-op\n" +title "Deploy 3: no config change, should converge to a no-op" trace $CLI bundle plan -title "Now the user removes spot_bid_max_price again\n" +title "Now the user removes spot_bid_max_price again" update_file.py databricks.yml "spot_bid_max_price: 0" "#spot_bid_max_price: 0" -title "Deploy 4: removing the explicit 0 is a change, symmetric with setting it\n" +title "Deploy 4: removing the explicit 0 is a change, symmetric with setting it" trace $CLI bundle plan -title "spot_bid_max_price change in the plan:\n" +title "spot_bid_max_price change in the plan:" $CLI bundle plan -o json | jq '.plan[].changes["azure_attributes.spot_bid_max_price"]' trace $CLI bundle deploy -qq -title "Deploy 5: no config change, should converge to a no-op\n" +title "Deploy 5: no config change, should converge to a no-op" trace $CLI bundle plan diff --git a/acceptance/bundle/resources/clusters/deploy/update-after-create/script b/acceptance/bundle/resources/clusters/deploy/update-after-create/script index 05409a7ffcb..b08f318f799 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-after-create/script +++ b/acceptance/bundle/resources/clusters/deploy/update-after-create/script @@ -7,11 +7,11 @@ trap cleanup EXIT trace $CLI bundle deploy -title "Cluster should exist after bundle deployment:\n" +title "Cluster should exist after bundle deployment:" CLUSTER_ID=$($CLI bundle summary -o json | jq -r '.resources.clusters.test_cluster.id') $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers}' -title "Updating cluster should call update API\n" +title "Updating cluster should call update API" # The cluster routes through the shared instance pool (node type c5d.xlarge), # whose maximum executor heap is 3157 MB, so keep the bumped value under that. update_file.py databricks.yml "\"spark.executor.memory\": \"2g\"" "\"spark.executor.memory\": \"3g\"" diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script index e336ebc564f..b34f9554b27 100755 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script @@ -13,15 +13,15 @@ $CLI bundle plan > out.plan_.txt $CLI bundle plan -o json | nostamp > out.plan_.json trace $CLI bundle deploy -q -title "Cluster should exist with num_workers after bundle deployment:\n" +title "Cluster should exist with num_workers after bundle deployment:" CLUSTER_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.clusters.test_cluster.id') add_repl "$CLUSTER_ID" CLUSTER_ID $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,autoscale}' -title "Terminating the cluster\n" +title "Terminating the cluster" $CLI clusters delete "${CLUSTER_ID}" | jq '{cluster_id}' -title "Adding autoscale section should call update API on stopped cluster\n" +title "Adding autoscale section should call update API on stopped cluster" update_file.py databricks.yml " num_workers: 2" " autoscale: min_workers: 2 max_workers: 4" @@ -31,10 +31,10 @@ trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' "$OUT_REQUESTS" rm "$OUT_REQUESTS" -title "Cluster should have autoscale\n" +title "Cluster should have autoscale" $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,autoscale}' -title "Changing autoscale should call update API on stopped cluster\n" +title "Changing autoscale should call update API on stopped cluster" update_file.py databricks.yml "min_workers: 2" "min_workers: 3" update_file.py databricks.yml "max_workers: 4" "max_workers: 5" $CLI bundle plan >> out.plan_.txt @@ -43,13 +43,13 @@ trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' "$OUT_REQUESTS" rm "$OUT_REQUESTS" -title "Cluster should have new autoscale\n" +title "Cluster should have new autoscale" $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,autoscale}' -title "Starting the cluster\n" +title "Starting the cluster" $CLI clusters start "${CLUSTER_ID}" -title "Changing autoscale should call resize API on running cluster\n" +title "Changing autoscale should call resize API on running cluster" update_file.py databricks.yml "min_workers: 3" "min_workers: 4" update_file.py databricks.yml "max_workers: 5" "max_workers: 6" $CLI bundle plan >> out.plan_.txt @@ -58,10 +58,10 @@ trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' "$OUT_REQUESTS" rm "$OUT_REQUESTS" -title "Cluster should have new autoscale\n" +title "Cluster should have new autoscale" $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,autoscale}' -title "Removing autoscale section should call resize API\n" +title "Removing autoscale section should call resize API" update_file.py databricks.yml " autoscale: min_workers: 4 max_workers: 6" " num_workers: 3" @@ -71,5 +71,5 @@ trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' "$OUT_REQUESTS" rm "$OUT_REQUESTS" -title "Cluster should have num_workers\n" +title "Cluster should have num_workers" $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,autoscale}' diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script index 06d5ae378cc..f6a053f1126 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script @@ -13,12 +13,12 @@ $CLI bundle plan > out.plan_.txt $CLI bundle plan -o json | nostamp > out.plan_.json trace $CLI bundle deploy -q -title "Cluster should exist after bundle deployment:\n" +title "Cluster should exist after bundle deployment:" CLUSTER_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.clusters.test_cluster.id') add_repl "$CLUSTER_ID" CLUSTER_ID $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers}' -title "Changing num_workers should call update API on stopped cluster\n" +title "Changing num_workers should call update API on stopped cluster" update_file.py databricks.yml "num_workers: 2" "num_workers: 3" $CLI bundle plan >> out.plan_.txt $CLI bundle plan -o json | nostamp >> out.plan_.json @@ -26,13 +26,13 @@ trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' "$OUT_REQUESTS" rm "$OUT_REQUESTS" -title "Cluster should have new num_workers\n" +title "Cluster should have new num_workers" $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers}' -title "Starting the cluster\n" +title "Starting the cluster" $CLI clusters start "${CLUSTER_ID}" -title "Changing num_workers should call resize API on running cluster\n" +title "Changing num_workers should call resize API on running cluster" update_file.py databricks.yml "num_workers: 3" "num_workers: 4" $CLI bundle plan >> out.plan_.txt $CLI bundle plan -o json | nostamp >> out.plan_.json @@ -40,10 +40,10 @@ trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' "$OUT_REQUESTS" rm "$OUT_REQUESTS" -title "Cluster should have new num_workers\n" +title "Cluster should have new num_workers" $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers}' -title "Changing num_workers and spark_conf should call update API\n" +title "Changing num_workers and spark_conf should call update API" update_file.py databricks.yml "num_workers: 4" "num_workers: 5" update_file.py databricks.yml '"spark.executor.memory": "2g"' '"spark.executor.memory": "4g"' $CLI bundle plan >> out.plan_.txt @@ -52,5 +52,5 @@ trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' "$OUT_REQUESTS" rm "$OUT_REQUESTS" -title "Cluster should have new num_workers and spark_conf\n" +title "Cluster should have new num_workers and spark_conf" $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,spark_conf}' diff --git a/acceptance/bundle/resources/clusters/deploy/use_preemptible_update/script b/acceptance/bundle/resources/clusters/deploy/use_preemptible_update/script index d2598b7a7aa..4b3ba64068b 100644 --- a/acceptance/bundle/resources/clusters/deploy/use_preemptible_update/script +++ b/acceptance/bundle/resources/clusters/deploy/use_preemptible_update/script @@ -4,34 +4,34 @@ cleanup() { } trap cleanup EXIT -title "Deploy 1: create the cluster with gcp_attributes but no use_preemptible_executors\n" +title "Deploy 1: create the cluster with gcp_attributes but no use_preemptible_executors" trace $CLI bundle deploy -qq -title "gcp_attributes sent on create (use_preemptible_executors absent):\n" +title "gcp_attributes sent on create (use_preemptible_executors absent):" print_requests.py //clusters/create | jq '.body.gcp_attributes' -title "Now the user adds an explicit use_preemptible_executors: false to the config\n" +title "Now the user adds an explicit use_preemptible_executors: false to the config" update_file.py databricks.yml "#use_preemptible_executors: false" "use_preemptible_executors: false" -title "Deploy 2: bundle should detect and apply the use_preemptible_executors change\n" +title "Deploy 2: bundle should detect and apply the use_preemptible_executors change" trace $CLI bundle plan -title "use_preemptible_executors change in the plan:\n" +title "use_preemptible_executors change in the plan:" $CLI bundle plan -o json | jq '.plan[].changes["gcp_attributes.use_preemptible_executors"]' trace $CLI bundle deploy -qq -title "gcp_attributes sent on update (empty means the change was NOT applied):\n" +title "gcp_attributes sent on update (empty means the change was NOT applied):" print_requests.py //clusters/edit | jq '.body.gcp_attributes' -title "Deploy 3: no config change, should converge to a no-op\n" +title "Deploy 3: no config change, should converge to a no-op" trace $CLI bundle plan -title "Now the user removes use_preemptible_executors again\n" +title "Now the user removes use_preemptible_executors again" update_file.py databricks.yml "use_preemptible_executors: false" "#use_preemptible_executors: false" -title "Deploy 4: removing the explicit false is a change, symmetric with setting it\n" +title "Deploy 4: removing the explicit false is a change, symmetric with setting it" trace $CLI bundle plan -title "use_preemptible_executors change in the plan:\n" +title "use_preemptible_executors change in the plan:" $CLI bundle plan -o json | jq '.plan[].changes["gcp_attributes.use_preemptible_executors"]' trace $CLI bundle deploy -qq -title "Deploy 5: no config change, should converge to a no-op\n" +title "Deploy 5: no config change, should converge to a no-op" trace $CLI bundle plan diff --git a/acceptance/bundle/resources/clusters/libraries-content-change/output.txt b/acceptance/bundle/resources/clusters/libraries-content-change/output.txt index ef3f8cf7c11..7b5146ee4f3 100644 --- a/acceptance/bundle/resources/clusters/libraries-content-change/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-content-change/output.txt @@ -1,5 +1,6 @@ === Deploy a cluster with a local wheel library + OK: cluster created with wheel >>> print_requests.py //libraries/install @@ -17,6 +18,7 @@ OK: cluster created with wheel } === Badness: wheel code changes but version does not, so redeploy does not reinstall it + >>> jq .plan["resources.clusters.mycluster"].action tmp.plan.json "skip" diff --git a/acceptance/bundle/resources/clusters/libraries-repo/output.txt b/acceptance/bundle/resources/clusters/libraries-repo/output.txt index 03dfa68dc62..003604056e1 100644 --- a/acceptance/bundle/resources/clusters/libraries-repo/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-repo/output.txt @@ -1,12 +1,15 @@ === Deploy creates the cluster and installs the pypi library + OK: cluster created with pypi library === Plan is a no-op after deploy: the repo round-trips + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Changing only the repo is planned as an update + >>> [CLI] bundle plan update clusters.mycluster diff --git a/acceptance/bundle/resources/clusters/libraries-scenarios/output.txt b/acceptance/bundle/resources/clusters/libraries-scenarios/output.txt index c31dc73b484..debe00e1d5e 100644 --- a/acceptance/bundle/resources/clusters/libraries-scenarios/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-scenarios/output.txt @@ -1,8 +1,10 @@ === Deploy a cluster that declares no libraries + OK: cluster created === Out of band: a job run (or the UI) installs 'six' on the shared cluster + >>> print_requests.py //libraries/install { "method": "POST", @@ -20,6 +22,7 @@ OK: cluster created } === No libraries section: six is skipped as unmanaged, not drift + >>> plan_libs { "libraries": { @@ -36,6 +39,7 @@ OK: cluster created } === Declare 'requests': plan installs it and uninstalls the out-of-band six + >>> plan_libs { "libraries": { @@ -104,6 +108,7 @@ OK: cluster created } === Drift: 'requests' is uninstalled out of band; plan reinstalls the declared library + >>> print_requests.py //libraries/uninstall { "method": "POST", @@ -158,6 +163,7 @@ OK: cluster created } === Remove the libraries section: libraries become unmanaged, requests stays installed, no install or uninstall + >>> plan_libs { "libraries": { @@ -183,6 +189,7 @@ OK: cluster created >>> print_requests.py //libraries/uninstall //libraries/install === No-op redeploy: libraries are unmanaged and nothing changes + >>> plan_libs { "libraries": { diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt index d7913bf1bb0..3922fcbc652 100644 --- a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt @@ -1,8 +1,10 @@ === Deploy libraries on a started=false cluster + OK: created on stopped cluster === Change the libraries and redeploy the stopped cluster + OK: libraries changed on stopped cluster >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/clusters/libraries-update/output.txt b/acceptance/bundle/resources/clusters/libraries-update/output.txt index 64c9ac77476..c1763ab3bfe 100644 --- a/acceptance/bundle/resources/clusters/libraries-update/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-update/output.txt @@ -1,5 +1,6 @@ === Create: the declared library installs on the new cluster + >>> print_requests.py //libraries/install { "method": "POST", @@ -17,6 +18,7 @@ } === Update: adding a library installs the declared set on the running cluster + >>> print_requests.py //libraries/install //libraries/uninstall { "method": "POST", @@ -39,6 +41,7 @@ } === Delete: emptying the libraries list uninstalls them + >>> print_requests.py //libraries/uninstall //libraries/install { "method": "POST", diff --git a/acceptance/bundle/resources/clusters/libraries/output.txt b/acceptance/bundle/resources/clusters/libraries/output.txt index 6e0dc533ca0..4d958c40a1b 100644 --- a/acceptance/bundle/resources/clusters/libraries/output.txt +++ b/acceptance/bundle/resources/clusters/libraries/output.txt @@ -1,5 +1,6 @@ === Deploy a cluster with a pypi and a local wheel library + >>> jq .plan["resources.clusters.mycluster"].new_state.value.libraries tmp.plan.json [ { @@ -15,6 +16,7 @@ OK: cluster created with libraries === Libraries installed via the Libraries API (wheel rewritten to its uploaded path) + >>> print_requests.py //libraries/install { "method": "POST", @@ -35,6 +37,7 @@ OK: cluster created with libraries } === Removing the wheel and redeploying uninstalls it + >>> print_requests.py //libraries/uninstall { "method": "POST", diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt index 1f0daa04149..63dadb67400 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt +++ b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt @@ -1,5 +1,6 @@ === Deploy with started=false: cluster created and then stopped + >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created clusters.mycluster @@ -30,6 +31,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "TERMINATED" === Toggle started=false -> started=true: only Start should be called, no Edit + >>> update_file.py databricks.yml started: false started: true >>> errcode [CLI] bundle deploy @@ -51,6 +53,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "RUNNING" === Toggle started=true -> started=false: only Stop should be called, no Edit + >>> update_file.py databricks.yml started: true started: false >>> errcode [CLI] bundle deploy diff --git a/acceptance/bundle/resources/clusters/lifecycle-started/output.txt b/acceptance/bundle/resources/clusters/lifecycle-started/output.txt index 264c8a4ca31..df9d1e30eb9 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started/output.txt +++ b/acceptance/bundle/resources/clusters/lifecycle-started/output.txt @@ -1,5 +1,6 @@ === Deploy with started=true: cluster created and running + >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created clusters.mycluster @@ -23,6 +24,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "RUNNING" === Stop cluster externally while config says started=true: plan detects drift + >>> errcode [CLI] clusters delete [UUID] "TERMINATED" @@ -32,6 +34,7 @@ update clusters.mycluster Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged === Deploy fixes the drift: cluster restarted + >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Updated clusters.mycluster @@ -58,6 +61,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "RUNNING" === Deploy with started=false while cluster is stopped: cluster stays terminated + >>> errcode [CLI] clusters delete [UUID] "TERMINATED" @@ -81,6 +85,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged "TERMINATED" === Deploy with started=true: cluster restarted + >>> update_file.py databricks.yml started: false started: true >>> errcode [CLI] bundle deploy diff --git a/acceptance/bundle/resources/clusters/readplan-lifecycle/output.txt b/acceptance/bundle/resources/clusters/readplan-lifecycle/output.txt index 83736aac03a..0b9cbc05eb0 100644 --- a/acceptance/bundle/resources/clusters/readplan-lifecycle/output.txt +++ b/acceptance/bundle/resources/clusters/readplan-lifecycle/output.txt @@ -1,5 +1,6 @@ === Initial deploy: cluster created and running + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created clusters.mycluster @@ -9,6 +10,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //clusters/start === Change tags and deploy from plan: no extra start + >>> update_file.py databricks.yml env: original env: updated Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Updated clusters.mycluster diff --git a/acceptance/bundle/resources/clusters/resize-terminated-fallback/script b/acceptance/bundle/resources/clusters/resize-terminated-fallback/script index 72399f2e419..e8780a28bb7 100644 --- a/acceptance/bundle/resources/clusters/resize-terminated-fallback/script +++ b/acceptance/bundle/resources/clusters/resize-terminated-fallback/script @@ -11,17 +11,17 @@ trace $CLI bundle deploy CLUSTER_ID=$($CLI bundle summary -o json | jq -r '.resources.clusters.test_cluster.id') add_repl "$CLUSTER_ID" CLUSTER_ID -title "Create a plan while cluster is running: should show resize\n" +title "Create a plan while cluster is running: should show resize" update_file.py databricks.yml "num_workers: 2" "num_workers: 3" trace $CLI bundle plan -o json > plan.json -title "Terminate the cluster before applying the saved plan\n" +title "Terminate the cluster before applying the saved plan" $CLI clusters delete "$CLUSTER_ID" > /dev/null trace $CLI clusters get "$CLUSTER_ID" | jq '{cluster_name,num_workers,state}' -title "Apply saved plan: resize fails with INVALID_STATE, falls back to edit\n" +title "Apply saved plan: resize fails with INVALID_STATE, falls back to edit" trace $CLI bundle deploy --plan plan.json trace print_requests.py //clusters/resize //clusters/edit -title "Cluster should have new num_workers\n" +title "Cluster should have new num_workers" trace $CLI clusters get "$CLUSTER_ID" | jq '{cluster_name,num_workers}' diff --git a/acceptance/bundle/resources/dashboards/change-serialized-inline/script b/acceptance/bundle/resources/dashboards/change-serialized-inline/script index f16a2584031..13a0c4a3d9a 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-inline/script +++ b/acceptance/bundle/resources/dashboards/change-serialized-inline/script @@ -28,7 +28,7 @@ trace $CLI bundle plan -o json > plan_raw.json # Generate golden file by applying nostamp for comparison (discard output) cat plan_raw.json | nostamp > out.plan.json -title "deploy the change\n" +title "deploy the change" $CLI bundle deploy $(readplanarg plan_raw.json) print_requests.py //api/2.0/lakeview/dashboards > out.requests.txt diff --git a/acceptance/bundle/resources/dashboards/detect-change/output.txt b/acceptance/bundle/resources/dashboards/detect-change/output.txt index 06d6548f11e..5b34273e875 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/output.txt +++ b/acceptance/bundle/resources/dashboards/detect-change/output.txt @@ -1,5 +1,6 @@ === Check summary before deployment -- should not have id but should have modified_status=created + >>> [CLI] bundle summary --output json { "display_name": "test-dashboard-[UNIQUE_NAME]", @@ -17,6 +18,7 @@ Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Assert that the dashboard exists at the expected path and is, indeed, a dashboard: + >>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/resources/test-dashboard-[UNIQUE_NAME].lvdash.json { "object_type": "DASHBOARD", @@ -24,6 +26,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Check summary again -- should have id but no modified_status + >>> [CLI] bundle summary --output json >>> cat out.summary.json @@ -38,7 +41,8 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" } -=== Load the dashboard by its ID and confirm its display name: { +=== Load the dashboard by its ID and confirm its display name: +{ "display_name": "test-dashboard-[UNIQUE_NAME]", "page_display_name": "New Page" } @@ -49,6 +53,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Try to redeploy the bundle and confirm that the out of band modification is detected: + >>> [CLI] bundle plan Warning: dashboard "file_reference" has been modified remotely at resources.dashboards.file_reference @@ -100,6 +105,7 @@ The remote modifications will be lost. Exit code: 1 === Redeploy the bundle with the --force flag and confirm that the out of band modification is ignored: + >>> errcode [CLI] bundle deploy --force Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Updated dashboards.file_reference diff --git a/acceptance/bundle/resources/dashboards/detect-change/script b/acceptance/bundle/resources/dashboards/detect-change/script index cdc1d201a15..a4e28041082 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/script +++ b/acceptance/bundle/resources/dashboards/detect-change/script @@ -29,7 +29,7 @@ rm out.summary.json title "Load the dashboard by its ID and confirm its display name: " echo "$DASHBOARD" | jq '{display_name,page_display_name: (.serialized_dashboard | fromjson | .pages[0].displayName)}' -title "Make an out of band modification to the dashboard and confirm that it is detected:\n" +title "Make an out of band modification to the dashboard and confirm that it is detected:" RESOURCE_ID=$($CLI workspace get-status "${DASHBOARD_PATH}" | jq -r '.resource_id') DASHBOARD_JSON="{\"serialized_dashboard\": \"{}\", \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\"}" UPDATE_RESP=$($CLI lakeview update "${RESOURCE_ID}" --json "${DASHBOARD_JSON}") diff --git a/acceptance/bundle/resources/dashboards/generate_inplace/script b/acceptance/bundle/resources/dashboards/generate_inplace/script index a8112a752a0..6d0a596e4c1 100644 --- a/acceptance/bundle/resources/dashboards/generate_inplace/script +++ b/acceptance/bundle/resources/dashboards/generate_inplace/script @@ -11,10 +11,10 @@ trap cleanup EXIT dashboard_id=$($CLI bundle summary -o json | jq .resources.dashboards.already_exists.id -r) -title "Get current dashboard\n" +title "Get current dashboard" retry $CLI lakeview get $dashboard_id | jq .serialized_dashboard -title "Update dashboard\n" +title "Update dashboard" $CLI lakeview update $dashboard_id --display-name "Updated dashboard" | jq .serialized_dashboard # This should error. diff --git a/acceptance/bundle/resources/dashboards/republish-after-draft-update/output.txt b/acceptance/bundle/resources/dashboards/republish-after-draft-update/output.txt index 4064ec231c7..3143f196b23 100644 --- a/acceptance/bundle/resources/dashboards/republish-after-draft-update/output.txt +++ b/acceptance/bundle/resources/dashboards/republish-after-draft-update/output.txt @@ -6,10 +6,12 @@ Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Plan right after deploy -- published is current, no change: + >>> [CLI] bundle plan -o json null === Plan after out-of-band draft update -- a republish is owed: + >>> [CLI] bundle plan -o json Warning: dashboard "dashboard1" has been modified remotely at resources.dashboards.dashboard1 diff --git a/acceptance/bundle/resources/data_quality_monitors/basic/output.txt b/acceptance/bundle/resources/data_quality_monitors/basic/output.txt index 2e3e7c287ad..ca2f863b50d 100644 --- a/acceptance/bundle/resources/data_quality_monitors/basic/output.txt +++ b/acceptance/bundle/resources/data_quality_monitors/basic/output.txt @@ -22,6 +22,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Summary shows the composite id + >>> [CLI] bundle summary -o json { "id": "schema,[UUID][0]", @@ -29,6 +30,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Verify deployment + >>> [CLI] data-quality get-monitor schema [UUID][0] { "excluded_table_full_names": [ @@ -37,10 +39,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === A second plan has no changes + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === Update the config (should update in place, not recreate) + >>> update_file.py databricks.yml main.myschema.table1 main.myschema.table2 >>> [CLI] bundle deploy @@ -75,6 +79,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } === Change an identity field (should plan a recreate) + >>> update_file.py databricks.yml ${resources.schemas.myschema.schema_id} [UUID][1] >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/data_quality_monitors/data_profiling/output.txt b/acceptance/bundle/resources/data_quality_monitors/data_profiling/output.txt index 3a6449d38f5..b3332745937 100644 --- a/acceptance/bundle/resources/data_quality_monitors/data_profiling/output.txt +++ b/acceptance/bundle/resources/data_quality_monitors/data_profiling/output.txt @@ -1,5 +1,6 @@ === Development mode drops the schedule (its pause_status is read-only) + >>> [CLI] bundle validate -o json -t dev null @@ -29,6 +30,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Verify deployment, including server-computed fields + >>> [CLI] data-quality get-monitor table [UUID][1] { "assets_dir": "/Workspace/Users/user@example.com/assets1", @@ -41,10 +43,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Server-computed fields do not show up as drift + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Update the schedule (should update in place) + >>> update_file.py databricks.yml 0 0 12 * * ? 0 0 6 * * ? >>> [CLI] bundle deploy @@ -76,6 +80,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Change assets_dir (ignored by the update API, should plan a recreate) + >>> update_file.py databricks.yml assets1 assets2 >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/data_quality_monitors/lifecycle/output.txt b/acceptance/bundle/resources/data_quality_monitors/lifecycle/output.txt index 739b8be4987..74f2506677d 100644 --- a/acceptance/bundle/resources/data_quality_monitors/lifecycle/output.txt +++ b/acceptance/bundle/resources/data_quality_monitors/lifecycle/output.txt @@ -7,10 +7,12 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === A second plan has no changes + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === Add an exclusion (should update in place) + >>> update_file.py databricks.yml anomaly_detection_config: {} anomaly_detection_config: {excluded_table_full_names: [main.dqm_[UNIQUE_NAME].excluded]} >>> [CLI] bundle deploy diff --git a/acceptance/bundle/resources/data_quality_monitors/remote-delete/output.txt b/acceptance/bundle/resources/data_quality_monitors/remote-delete/output.txt index bf119a1c9ba..960e5d3569f 100644 --- a/acceptance/bundle/resources/data_quality_monitors/remote-delete/output.txt +++ b/acceptance/bundle/resources/data_quality_monitors/remote-delete/output.txt @@ -6,9 +6,11 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Delete the monitor out of band + >>> [CLI] data-quality delete-monitor table [UUID] === Plan should detect the monitor is gone and re-create it + >>> [CLI] bundle plan create data_quality_monitors.table_monitor diff --git a/acceptance/bundle/resources/database_catalogs/recreate/output.txt b/acceptance/bundle/resources/database_catalogs/recreate/output.txt index 40b7bf2adec..8ea69a0292a 100644 --- a/acceptance/bundle/resources/database_catalogs/recreate/output.txt +++ b/acceptance/bundle/resources/database_catalogs/recreate/output.txt @@ -7,6 +7,7 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Change an immutable catalog field: no update API, so the plan recreates it + >>> [CLI] bundle plan recreate database_catalogs.my_catalog diff --git a/acceptance/bundle/resources/database_instances/recreate/output.txt b/acceptance/bundle/resources/database_instances/recreate/output.txt index dc6c194d76f..1753e7b135e 100644 --- a/acceptance/bundle/resources/database_instances/recreate/output.txt +++ b/acceptance/bundle/resources/database_instances/recreate/output.txt @@ -6,6 +6,7 @@ Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Rename the instance: name is immutable, so the plan recreates it + >>> [CLI] bundle plan recreate database_instances.my_database diff --git a/acceptance/bundle/resources/experiments/basic/output.txt b/acceptance/bundle/resources/experiments/basic/output.txt index fc5ad8ebb98..be16a709a17 100644 --- a/acceptance/bundle/resources/experiments/basic/output.txt +++ b/acceptance/bundle/resources/experiments/basic/output.txt @@ -1,5 +1,6 @@ === create the experiment + >>> export EXPERIMENT_ARTIFACT_LOCATION=s3://original-[UNIQUE_NAME] >>> export EXPERIMENT_NAME_PREFIX=original @@ -46,6 +47,7 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged } === update the name + >>> export EXPERIMENT_NAME_PREFIX=new-name >>> envsubst @@ -90,6 +92,7 @@ Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged } === updating the artifact location should cause a recreation + >>> export EXPERIMENT_ARTIFACT_LOCATION=s3://new-[UNIQUE_NAME] >>> envsubst @@ -134,6 +137,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged } === adding a new tag should be a no-op + >>> envsubst >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/external_locations/comment_cleared/output.txt b/acceptance/bundle/resources/external_locations/comment_cleared/output.txt index a7b39cb324d..0d755db1b38 100644 --- a/acceptance/bundle/resources/external_locations/comment_cleared/output.txt +++ b/acceptance/bundle/resources/external_locations/comment_cleared/output.txt @@ -1,5 +1,6 @@ === Deploy with a comment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created external_locations.location1 @@ -7,6 +8,7 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Remove the comment from the configuration and redeploy + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Updated external_locations.location1 @@ -14,14 +16,17 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === The comment the bundle no longer declares + >>> [CLI] api get /api/2.1/unity-catalog/external-locations/myloc "" === Redeploy + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The UpdateExternalLocation payloads the deploys sent + >>> print_requests.py --method PATCH //external-locations { "method": "PATCH", diff --git a/acceptance/bundle/resources/external_locations/output.txt b/acceptance/bundle/resources/external_locations/output.txt index 56858719e8e..e2e6c9de660 100644 --- a/acceptance/bundle/resources/external_locations/output.txt +++ b/acceptance/bundle/resources/external_locations/output.txt @@ -1,5 +1,6 @@ === Deploy bundle with catalog and external location + >>> [CLI] bundle plan create catalogs.test_catalog create catalogs.test_catalog.grants @@ -18,6 +19,7 @@ Files: 5 uploaded, 0 deleted Resources: 4 created, 0 changed, 0 deleted, 0 unchanged === Assert the catalog is created with grants + >>> [CLI] catalogs get test_catalog_[UNIQUE_NAME] { "name": "test_catalog_[UNIQUE_NAME]", @@ -41,6 +43,7 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged } === Assert the external location is created with grants + >>> [CLI] external-locations get test_ext_location_[UNIQUE_NAME] { "name": "test_ext_location_[UNIQUE_NAME]", @@ -63,7 +66,9 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged } === Update external location comment + === Redeploy with updated comment + >>> [CLI] bundle plan update external_locations.test_location @@ -76,6 +81,7 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 3 unchanged === Assert the external location comment is updated + >>> [CLI] external-locations get test_ext_location_[UNIQUE_NAME] { "name": "test_ext_location_[UNIQUE_NAME]", @@ -83,7 +89,9 @@ Resources: 0 created, 1 changed, 0 deleted, 3 unchanged } === Update catalog comment + === Redeploy with updated catalog comment + >>> [CLI] bundle plan update catalogs.test_catalog @@ -96,6 +104,7 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 3 unchanged === Assert the catalog comment is updated + >>> [CLI] catalogs get test_catalog_[UNIQUE_NAME] { "name": "test_catalog_[UNIQUE_NAME]", @@ -103,6 +112,7 @@ Resources: 0 created, 1 changed, 0 deleted, 3 unchanged } === Test cleanup + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.catalogs.test_catalog diff --git a/acceptance/bundle/resources/feature_specs/basic/output.txt b/acceptance/bundle/resources/feature_specs/basic/output.txt index 985a38a245b..e07ac627f59 100644 --- a/acceptance/bundle/resources/feature_specs/basic/output.txt +++ b/acceptance/bundle/resources/feature_specs/basic/output.txt @@ -7,6 +7,7 @@ >>> create_feature_spec_fixtures.py main.fspec_[UNIQUE_NAME] === Initial summary before deploy + >>> [CLI] bundle summary -o json { "extra_pip_requirements": { @@ -91,6 +92,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Summary should show the id and the Catalog Explorer url + >>> [CLI] bundle summary -o json { "id": "[SPEC1_ID]", @@ -98,10 +100,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === A second plan on an unchanged bundle must report no changes + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Change a field (every field is immutable, so this recreates) + >>> update_file.py databricks.yml [age, country] [age] >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/feature_specs/order/output.txt b/acceptance/bundle/resources/feature_specs/order/output.txt index ea08d118636..024250568df 100644 --- a/acceptance/bundle/resources/feature_specs/order/output.txt +++ b/acceptance/bundle/resources/feature_specs/order/output.txt @@ -13,10 +13,12 @@ Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Reorder the lists whose order means nothing: no changes + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Also reorder the lookup keys, which must follow the table's primary key: recreate + >>> [CLI] bundle plan recreate feature_specs.spec1 diff --git a/acceptance/bundle/resources/feature_specs/remote-delete/output.txt b/acceptance/bundle/resources/feature_specs/remote-delete/output.txt index 4fe99398c4b..4fdddf0b298 100644 --- a/acceptance/bundle/resources/feature_specs/remote-delete/output.txt +++ b/acceptance/bundle/resources/feature_specs/remote-delete/output.txt @@ -13,10 +13,12 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Delete the feature spec out of band + >>> [CLI] api delete /api/2.0/feature-engineering/feature-specs/main.fspec_[UNIQUE_NAME].myspec {} === Plan should detect the feature spec is gone and re-create it + >>> [CLI] bundle plan create feature_specs.spec1 diff --git a/acceptance/bundle/resources/feature_specs/uppercase-name/output.txt b/acceptance/bundle/resources/feature_specs/uppercase-name/output.txt index 67caf37b353..635eb197427 100644 --- a/acceptance/bundle/resources/feature_specs/uppercase-name/output.txt +++ b/acceptance/bundle/resources/feature_specs/uppercase-name/output.txt @@ -35,10 +35,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === UC lowercases the name, so the id is lowercase + >>> [CLI] bundle summary -o json "main.fspec_[UNIQUE_NAME].myspec" === A plan on an unchanged bundle skips the name instead of recreating the feature spec + >>> [CLI] bundle plan -o json { "resources.feature_specs.spec1": { @@ -56,6 +58,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Renaming the feature spec still recreates it + >>> update_file.py databricks.yml .MySpec .MyOtherSpec >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/features/basic/output.txt b/acceptance/bundle/resources/features/basic/output.txt index 37f3458c1fd..3b6dcedeadd 100644 --- a/acceptance/bundle/resources/features/basic/output.txt +++ b/acceptance/bundle/resources/features/basic/output.txt @@ -1,5 +1,6 @@ === Initial summary before deploy + >>> [CLI] bundle summary -o json { "description": "DESCRIPTION1", @@ -34,6 +35,7 @@ } === Verify it does not exist yet + >>> musterr [CLI] feature-engineering get-feature main.myschema.myfeature Error: Resource ml.Feature not found: main.myschema.myfeature @@ -80,6 +82,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Summary should show the id and the Catalog Explorer url + >>> [CLI] bundle summary -o json { "id": "main.myschema.myfeature", @@ -87,6 +90,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Verify deployment + >>> [CLI] feature-engineering get-feature main.myschema.myfeature { "full_name": "main.myschema.myfeature", @@ -97,6 +101,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Update description (should update in place, not recreate) + >>> update_file.py databricks.yml DESCRIPTION1 DESCRIPTION2 >>> [CLI] bundle deploy @@ -148,6 +153,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "DESCRIPTION2" === Change an immutable field (should recreate, and ask for approval first) + >>> update_file.py databricks.yml amount other_column >>> musterr [CLI] bundle deploy diff --git a/acceptance/bundle/resources/features/lifecycle/output.txt b/acceptance/bundle/resources/features/lifecycle/output.txt index fe31357c479..d898408816c 100644 --- a/acceptance/bundle/resources/features/lifecycle/output.txt +++ b/acceptance/bundle/resources/features/lifecycle/output.txt @@ -16,10 +16,12 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === A second plan on an unchanged bundle must report no changes + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Update the description in place, then confirm the plan settles again + >>> update_file.py databricks.yml DESCRIPTION1 DESCRIPTION2 >>> [CLI] bundle deploy diff --git a/acceptance/bundle/resources/features/remote-delete/output.txt b/acceptance/bundle/resources/features/remote-delete/output.txt index 3047716268a..9ff9b42f708 100644 --- a/acceptance/bundle/resources/features/remote-delete/output.txt +++ b/acceptance/bundle/resources/features/remote-delete/output.txt @@ -6,9 +6,11 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Delete the feature out of band + >>> [CLI] feature-engineering delete-feature main.myschema.myfeature === Plan should detect the feature is gone and re-create it + >>> [CLI] bundle plan create features.feature1 diff --git a/acceptance/bundle/resources/genie_spaces/change-serialized-space/output.txt b/acceptance/bundle/resources/genie_spaces/change-serialized-space/output.txt index 9100e42528f..ec65aa9cfd3 100644 --- a/acceptance/bundle/resources/genie_spaces/change-serialized-space/output.txt +++ b/acceptance/bundle/resources/genie_spaces/change-serialized-space/output.txt @@ -6,10 +6,12 @@ Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Re-plan should be a no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Plan after changing serialized_space should show update + >>> [CLI] bundle plan update genie_spaces.my_space diff --git a/acceptance/bundle/resources/genie_spaces/delete_warning/output.txt b/acceptance/bundle/resources/genie_spaces/delete_warning/output.txt index 52720a7fcda..b0562268af7 100644 --- a/acceptance/bundle/resources/genie_spaces/delete_warning/output.txt +++ b/acceptance/bundle/resources/genie_spaces/delete_warning/output.txt @@ -6,6 +6,7 @@ Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Deploying without the genie space warns before deleting it (chat history loss) + >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/genie-delete-warning-[UNIQUE_NAME]/default/files... diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_update/output.txt b/acceptance/bundle/resources/genie_spaces/parent_path_update/output.txt index 1d878f1f874..0d7ebf6c9ca 100644 --- a/acceptance/bundle/resources/genie_spaces/parent_path_update/output.txt +++ b/acceptance/bundle/resources/genie_spaces/parent_path_update/output.txt @@ -6,6 +6,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Plan after changing parent_path should show update + >>> [CLI] bundle plan update genie_spaces.update_target diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/output.txt b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/output.txt index d346a1d8da3..23e42521d99 100644 --- a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/output.txt +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/output.txt @@ -8,6 +8,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] genie trash-space [GENIE_SPACE_ID] === Plan after remote deletion should recreate the space + >>> [CLI] bundle plan create genie_spaces.sales_analytics diff --git a/acceptance/bundle/resources/genie_spaces/version_migration/output.txt b/acceptance/bundle/resources/genie_spaces/version_migration/output.txt index 8b764650fcb..3aa8fdc982d 100644 --- a/acceptance/bundle/resources/genie_spaces/version_migration/output.txt +++ b/acceptance/bundle/resources/genie_spaces/version_migration/output.txt @@ -6,10 +6,12 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Plan after deploy is drift-free despite the version 1 -> version 2 migration + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged -=== serialized_space: local version 1, remote version 2, skipped (etag_based){ +=== serialized_space: local version 1, remote version 2, skipped (etag_based) +{ "action": "skip", "reason": "etag_based", "local_version": 1, diff --git a/acceptance/bundle/resources/grants/feature_specs/output.txt b/acceptance/bundle/resources/grants/feature_specs/output.txt index 3a9978724f2..a7b6b884b83 100644 --- a/acceptance/bundle/resources/grants/feature_specs/output.txt +++ b/acceptance/bundle/resources/grants/feature_specs/output.txt @@ -32,6 +32,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Every change recreates the feature spec, which drops its grants; the bundle must re-apply them + >>> update_file.py databricks.yml [age, country] [age] >>> [CLI] bundle plan -o json diff --git a/acceptance/bundle/resources/grants/schemas/change_name/output.txt b/acceptance/bundle/resources/grants/schemas/change_name/output.txt index 6c4e069ee0c..1cc43885cdf 100644 --- a/acceptance/bundle/resources/grants/schemas/change_name/output.txt +++ b/acceptance/bundle/resources/grants/schemas/change_name/output.txt @@ -1,5 +1,6 @@ === Deploy a schema with grants + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -38,6 +39,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged schemas schema1.grants id='schema/main.test-schema-[UNIQUE_NAME]' === Rename the schema: its ID changes, and so does the ID of its grants + >>> update_file.py databricks.yml test-schema- test-schema-renamed- >>> [CLI] bundle plan --output json @@ -52,6 +54,7 @@ schemas schema1.grants id='schema/main.test-schema-[UNIQUE_NAME]' } === Deploy moves the schema to the new name and applies the grants there + >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... @@ -105,6 +108,7 @@ Resources: 1 created, 1 changed, 1 deleted, 0 unchanged } === The grants are recorded under the new name, so the next plan is a no-op + >>> read_state.py schemas schema1.grants id schemas schema1.grants id='schema/main.test-schema-renamed-[UNIQUE_NAME]' diff --git a/acceptance/bundle/resources/grants/volume_change_name/output.txt b/acceptance/bundle/resources/grants/volume_change_name/output.txt index ffe41036393..8537be1c256 100644 --- a/acceptance/bundle/resources/grants/volume_change_name/output.txt +++ b/acceptance/bundle/resources/grants/volume_change_name/output.txt @@ -1,5 +1,6 @@ === Deploy a volume with grants + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -49,6 +50,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged volumes volume1.grants id='volume/main.test-schema-[UNIQUE_NAME].myvolume' === Rename the volume: its ID changes, and so does the ID of its grants + >>> update_file.py databricks.yml myvolume myvolume_renamed >>> [CLI] bundle plan --output json @@ -63,6 +65,7 @@ volumes volume1.grants id='volume/main.test-schema-[UNIQUE_NAME].myvolume' } === Deploy renames the volume and applies the grants under the new name + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated volumes.volume1 @@ -106,6 +109,7 @@ Resources: 0 created, 2 changed, 0 deleted, 1 unchanged } === The grants are recorded under the new name, so the next plan is a no-op + >>> read_state.py volumes volume1.grants id volumes volume1.grants id='volume/main.test-schema-[UNIQUE_NAME].myvolume_renamed' diff --git a/acceptance/bundle/resources/instance_pools/output.txt b/acceptance/bundle/resources/instance_pools/output.txt index 950c7d70287..bec110891ac 100644 --- a/acceptance/bundle/resources/instance_pools/output.txt +++ b/acceptance/bundle/resources/instance_pools/output.txt @@ -38,6 +38,7 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Verify the create request + >>> jq select(.method == "POST" and (.path | contains("/instance-pools/create"))) [OUTPUT_DIR]/out.requests.txt { "method": "POST", @@ -64,6 +65,7 @@ Resources: URL: [DATABRICKS_URL]/compute/instance-pools/[UUID]?w=[WORKSPACE_ID] === Update the instance pool name + >>> update_file.py databricks.yml my_instance_pool my_instance_pool_2 >>> [CLI] bundle deploy @@ -73,6 +75,7 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Verify the update request + >>> jq select(.method == "POST" and (.path | contains("/instance-pools/edit"))) [OUTPUT_DIR]/out.requests.txt { "method": "POST", @@ -100,6 +103,7 @@ Resources: URL: [DATABRICKS_URL]/compute/instance-pools/[UUID]?w=[WORKSPACE_ID] === Destroy the instance pool + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.instance_pools.test_instance_pool @@ -109,6 +113,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted === Verify the destroy request + >>> jq select(.method == "POST" and (.path | contains("/instance-pools/delete"))) [OUTPUT_DIR]/out.requests.txt { "method": "POST", diff --git a/acceptance/bundle/resources/job_runs/basic/output.txt b/acceptance/bundle/resources/job_runs/basic/output.txt index ba8bef3a0c8..fb740e217b6 100644 --- a/acceptance/bundle/resources/job_runs/basic/output.txt +++ b/acceptance/bundle/resources/job_runs/basic/output.txt @@ -1,5 +1,6 @@ === deploy triggers exactly one run + >>> [CLI] bundle validate Name: job-runs-basic Target: default @@ -65,6 +66,7 @@ Resources: URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]?w=[WORKSPACE_ID] === exactly one run-now request was made + >>> print_requests.py //jobs/run-now { "method": "POST", @@ -76,6 +78,7 @@ Resources: } === the triggered run id is stored in state, without result_state + >>> read_state.py job_runs my_run id result_state job_runs my_run id='[MY_RUN_ID]' result_state=None diff --git a/acceptance/bundle/resources/job_runs/destroy_unfinished_run/output.txt b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/output.txt index 6288dbd63c0..7903902a6cc 100644 --- a/acceptance/bundle/resources/job_runs/destroy_unfinished_run/output.txt +++ b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/output.txt @@ -1,5 +1,6 @@ === the deploy stops waiting before the run finishes + >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-destroy-unfinished-run/default/files... Created jobs.my_job @@ -18,6 +19,7 @@ Exit code: 1 [MY_RUN_ID] === destroy leaves the unfinished run alone + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.my_job diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index 89861c025a4..9f2dd59d0cf 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -1,5 +1,6 @@ === a run that finishes FAILED fails the deploy + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created jobs.my_job @@ -13,6 +14,7 @@ Error: cannot create resources.jobs.downstream_job: dependency failed: resources Files: 3 uploaded, 0 deleted === the failed run is recorded, and not having succeeded is drift + >>> read_id.py my_run [MY_RUN_ID] @@ -55,6 +57,7 @@ Files: 3 uploaded, 0 deleted } === so a redeploy runs the job again, and fails again + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [RUN_URL] @@ -67,6 +70,7 @@ Error: cannot create resources.jobs.downstream_job: dependency failed: resources Files: 0 uploaded, 0 deleted === run-now was issued once per deploy, and the recreate left the failed run alone + >>> print_requests.py --keep //jobs/run-now { "method": "POST", diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt index 7b911a49c38..45488a28b57 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt +++ b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt @@ -1,5 +1,6 @@ === the deploy stops waiting before the run finishes + >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-interrupted-run/default/files... Created jobs.my_job @@ -15,6 +16,7 @@ Files: 1 uploaded, 0 deleted Exit code: 1 === the abandoned run is still tracked + >>> read_id.py my_job [MY_JOB_ID] @@ -32,6 +34,7 @@ Exit code: 1 } === a run that is still going is skipped + >>> jq .plan["resources.job_runs.my_run"] tmp.plan.json { "depends_on": [ @@ -61,12 +64,14 @@ Exit code: 1 } === Badness: the next deploy does not wait for that run + >>> [CLI] bundle deploy --plan tmp.plan.json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-interrupted-run/default/files... Files: 1 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 2 unchanged === the same run is still tracked, and nothing was cancelled, deleted or triggered + >>> read_state.py job_runs my_run id result_state job_runs my_run id='[MY_RUN_ID]' result_state=None diff --git a/acceptance/bundle/resources/job_runs/job_parameters/output.txt b/acceptance/bundle/resources/job_runs/job_parameters/output.txt index d3c5ff16368..c43ab972833 100644 --- a/acceptance/bundle/resources/job_runs/job_parameters/output.txt +++ b/acceptance/bundle/resources/job_runs/job_parameters/output.txt @@ -1,5 +1,6 @@ === deploy triggers the run with only the overridden parameter + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-job-parameters/default/files... Created jobs.my_job @@ -23,6 +24,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === plan is stable: the resolved job default does not look like drift + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt index 50e7e4a46d7..2b79d541870 100644 --- a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt @@ -1,5 +1,6 @@ === first deploy triggers a run + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-bundle-deploy/default/files... Created jobs.my_job @@ -23,6 +24,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === redeploy re-fires with unchanged config + >>> [CLI] bundle plan recreate job_runs.my_run @@ -69,6 +71,7 @@ Resources: URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]?w=[WORKSPACE_ID] === second run-now after recreate + >>> print_requests.py --keep //jobs/runs/delete >>> print_requests.py //jobs/run-now @@ -82,6 +85,7 @@ Resources: } === removing on_bundle_deploy is unchanged and does not run + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/script b/acceptance/bundle/resources/job_runs/on_bundle_deploy/script index 10f95d176f4..cd4a58bab4e 100644 --- a/acceptance/bundle/resources/job_runs/on_bundle_deploy/script +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/script @@ -17,7 +17,7 @@ trace $CLI bundle plan # The deploys are not traced: readplanarg makes the command line differ per variant. $CLI bundle plan -o json > tmp.plan.json trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json -title "bundle deploy\n" +title "bundle deploy" $CLI bundle deploy $(readplanarg tmp.plan.json) read_id.py my_run > /dev/null trace $CLI bundle summary @@ -36,6 +36,6 @@ update_file.py databricks.yml " lifecycle: trace $CLI bundle plan $CLI bundle plan -o json > tmp.plan.json trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json -title "bundle deploy\n" +title "bundle deploy" $CLI bundle deploy $(readplanarg tmp.plan.json) trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index 529c8ed962a..987d638339d 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -1,5 +1,6 @@ === first deploy triggers a run + >>> [CLI] bundle plan -o json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... Created jobs.my_job @@ -23,6 +24,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === redeploy with unchanged file plans nothing + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... Files: 0 uploaded, 0 deleted @@ -31,6 +33,7 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now === touch without content change plans nothing + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... Files: 1 uploaded, 0 deleted @@ -39,6 +42,7 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now === editing the file re-fires + >>> jq .plan["resources.job_runs.my_run"] tmp.plan1.json { "depends_on": [ @@ -113,6 +117,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === deleting the file warns and re-fires + >>> [CLI] bundle plan -o json Warning: lifecycle.triggers.on_file_change: no synced files match "seed.txt" in databricks.yml:18:29 @@ -194,6 +199,7 @@ Warning: lifecycle.triggers.on_file_change: no synced files match "seed.txt" Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === restoring the file re-fires again + >>> [CLI] bundle plan -o json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_4]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_4]?o=[WORKSPACE_ID] @@ -216,6 +222,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === removing on_file_change is unchanged and does not run + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -230,6 +237,7 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === re-adding on_file_change does not re-fire an unchanged file + >>> print_state.py { "triggers_state": { diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index 6a3121450c8..2a7e23ac042 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -32,7 +32,7 @@ title "editing the file re-fires" update_file.py seed.txt "v1" "v2" $CLI bundle plan -o json > tmp.plan1.json trace jq '.plan["resources.job_runs.my_run"]' tmp.plan1.json -title "bundle deploy\n" +title "bundle deploy" $CLI bundle deploy $(readplanarg tmp.plan1.json) read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete @@ -63,7 +63,7 @@ update_file.py databricks.yml " lifecycle: " "" trace $CLI bundle plan $CLI bundle plan -o json > tmp.plan4.json -title "bundle deploy\n" +title "bundle deploy" $CLI bundle deploy $(readplanarg tmp.plan4.json) trace print_requests.py //jobs/run-now plan_no_changes diff --git a/acceptance/bundle/resources/job_runs/on_file_change_build/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_build/output.txt index e36e49bfa63..334bd0ad9c9 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_build/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_build/output.txt @@ -1,5 +1,6 @@ === first deploy triggers a run + >>> [CLI] bundle deploy Executing 'prebuild' script Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-build/default/files... @@ -24,6 +25,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === a prebuild file change re-fires in the same deploy + >>> [CLI] bundle deploy Executing 'prebuild' script Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-build/default/files... diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index 73f170fe8e9..fe7703423e7 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -1,5 +1,6 @@ === a pattern that escapes the sync root is rejected + >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root in databricks.yml:24:29 @@ -13,6 +14,7 @@ Workspace: Found 1 error === a glob that only matches directories warns + >>> [CLI] bundle validate Warning: lifecycle.triggers.on_file_change: no synced files match "dirs/*" in databricks.yml:24:29 @@ -26,6 +28,7 @@ Workspace: Found 1 warning === a glob matching a directory and an excluded file warns + >>> [CLI] bundle validate Warning: lifecycle.triggers.on_file_change: no synced files match "mixed/*" in databricks.yml:24:29 @@ -39,6 +42,7 @@ Workspace: Found 1 warning === a glob matching only excluded files warns + >>> [CLI] bundle validate Warning: lifecycle.triggers.on_file_change: no synced files match "allexcluded/*" in databricks.yml:24:29 @@ -52,6 +56,7 @@ Workspace: Found 1 warning === a pattern that matches nothing warns + >>> [CLI] bundle validate Warning: lifecycle.triggers.on_file_change: no synced files match "missing/*.txt" in databricks.yml:24:29 @@ -65,6 +70,7 @@ Workspace: Found 1 warning === ** is not supported + >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: ** in "src/**" is not supported; use * for a single directory level in databricks.yml:24:29 @@ -78,6 +84,7 @@ Workspace: Found 1 error === an absolute pattern is rejected + >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "/tmp/seed.txt" must be relative to the defining YAML file in databricks.yml:18:29 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 3f189586db7..8740172f2f7 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -1,5 +1,6 @@ === deploy hashes the matched files + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Created jobs.my_job @@ -127,10 +128,12 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } === editing a gitignored file does not re-fire + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === removing one pattern does not re-fire + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 04a951a4bc2..bc69fc1605a 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -19,34 +19,34 @@ trace read_id.py my_job read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -title "editing one matched file re-fires\n" +title "editing one matched file re-fires" update_file.py migrations/001.txt "one" "one-edited" changes # The deploy is not traced: readplanarg makes the command line differ per variant. -title "bundle deploy\n" +title "bundle deploy" $CLI bundle deploy $(readplanarg tmp.plan.json) read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now -title "editing a file in a subdirectory re-fires\n" +title "editing a file in a subdirectory re-fires" # printf keeps LF on Windows, where update_file.py's text-mode rewrite would # restore CRLF and leave the file hashing differently than the deploy above. printf 'sub-edited\n' > mysubdir/a.txt changes printf 'sub\n' > mysubdir/a.txt -title "a new matching file re-fires\n" +title "a new matching file re-fires" printf 'three\n' > migrations/003.txt changes rm migrations/003.txt -title "removing a matched file re-fires\n" +title "removing a matched file re-fires" mv migrations/002.txt tmp.002.txt changes mv tmp.002.txt migrations/002.txt -title "editing a file below the glob does not re-fire\n" +title "editing a file below the glob does not re-fire" # migrations/* matches the archive directory itself, which is skipped: the glob # does not descend into it. update_file.py migrations/archive/003.txt "three" "three-edited" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt index 664e42bf0c0..30abbdec577 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt @@ -1,5 +1,6 @@ === first deploy + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include-sync-root/default/files... Created jobs.my_job @@ -17,6 +18,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === editing the file next to the included YAML re-fires + >>> changes { "lifecycle.triggers_state.on_file_change['bundle/resources/watched-resources.txt']": { @@ -34,6 +36,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === editing the file at the bundle root re-fires + >>> changes { "lifecycle.triggers_state.on_file_change['bundle/watched-bundle.txt']": { @@ -51,6 +54,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === editing the file at the sync root re-fires + >>> changes { "lifecycle.triggers_state.on_file_change['shared.txt']": { diff --git a/acceptance/bundle/resources/job_runs/on_file_change_predeploy/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/output.txt index 5e4224e2469..48efc78706b 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_predeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/output.txt @@ -1,5 +1,6 @@ === first deploy triggers a run + >>> [CLI] bundle deploy Executing 'predeploy' script Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-predeploy/default/files... @@ -24,6 +25,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === a predeploy file change re-fires in the same deploy + >>> [CLI] bundle deploy Executing 'predeploy' script Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-predeploy/default/files... diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index c88a8a5d1b6..55c3a02f7cc 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -1,5 +1,6 @@ === initial deploy triggers the first run + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default/files... Created jobs.my_job @@ -42,6 +43,7 @@ Resources: } === change the run configuration and redeploy + >>> update_file.py databricks.yml env: dev env: prod >>> [CLI] bundle plan -o json @@ -119,6 +121,7 @@ Resources: URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]?w=[WORKSPACE_ID] === the config change triggered a second, different run and left the previous one alone + >>> print_requests.py --keep //jobs/runs/delete >>> print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/retried_run_now/output.txt b/acceptance/bundle/resources/job_runs/retried_run_now/output.txt index a856ca44fc9..70a9064dac1 100644 --- a/acceptance/bundle/resources/job_runs/retried_run_now/output.txt +++ b/acceptance/bundle/resources/job_runs/retried_run_now/output.txt @@ -1,5 +1,6 @@ === the deploy succeeds, though its run-now was answered with a 503 + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-retried-run-now/default/files... Created jobs.my_job @@ -10,6 +11,7 @@ Files: 1 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === run-now was sent twice, both times with the same token + >>> read_id.py my_job [MY_JOB_ID] @@ -32,6 +34,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === the second request was answered with the run the first one started + >>> read_id.py my_run [MY_RUN_ID] diff --git a/acceptance/bundle/resources/job_runs/wait/output.txt b/acceptance/bundle/resources/job_runs/wait/output.txt index 0d7596a9462..500a72abeb3 100644 --- a/acceptance/bundle/resources/job_runs/wait/output.txt +++ b/acceptance/bundle/resources/job_runs/wait/output.txt @@ -1,5 +1,6 @@ === the deploy waits for the run to finish + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created jobs.my_job @@ -14,10 +15,12 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged [MY_RUN_ID] === the downstream job was created with the run's result_state + >>> [CLI] jobs get [DOWNSTREAM_JOB_ID] -o json SUCCESS === nothing the run resolved is drift, so a redeploy starts no new run + >>> [CLI] bundle plan -o json resources.job_runs.my_run skip resources.jobs.downstream_job skip diff --git a/acceptance/bundle/resources/jobs/check-metadata/output.txt b/acceptance/bundle/resources/jobs/check-metadata/output.txt index fd85689b6ef..f882a70f928 100644 --- a/acceptance/bundle/resources/jobs/check-metadata/output.txt +++ b/acceptance/bundle/resources/jobs/check-metadata/output.txt @@ -1,5 +1,6 @@ === First summary, should not have id and modified_status should be created + >>> [CLI] bundle summary -o json { "id": null, @@ -14,6 +15,7 @@ Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Second summary, should have id and modified status should be absent + >>> [CLI] bundle summary -o json >>> cat out.summary.json @@ -33,6 +35,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Read metadata object from the workspace + >>> [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/metadata.json >>> cat metadata.json @@ -69,6 +72,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "extra": {} } === Check that metadata contains created job ids + >>> cat metadata.json "id": "[JOB_1_ID]", @@ -76,6 +80,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "id": "[JOB_2_ID]", === Check that metadata contains bundle name, target, and mode + >>> cat metadata.json "deploy-jobs-check-metadata-[UNIQUE_NAME]" diff --git a/acceptance/bundle/resources/jobs/check-metadata/script b/acceptance/bundle/resources/jobs/check-metadata/script index a486009c74c..6af4fe8ed7b 100644 --- a/acceptance/bundle/resources/jobs/check-metadata/script +++ b/acceptance/bundle/resources/jobs/check-metadata/script @@ -16,10 +16,10 @@ trace cat out.summary.json JOB_1_ID=$(jq -r '.id' out.summary.json) add_repl "$JOB_1_ID" JOB_1_ID rm out.summary.json -title "Assert job 1 is created\n" +title "Assert job 1 is created" $CLI jobs get "${JOB_1_ID}" | jq '{name: .settings.name}' -title "Assert job 2 is created\n" +title "Assert job 2 is created" JOB_2_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.bar.id') add_repl "$JOB_2_ID" JOB_2_ID $CLI jobs get "${JOB_2_ID}" | jq '{name: .settings.name}' diff --git a/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/script b/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/script index b8cb4a39cbf..f8004aa8af1 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/script +++ b/acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/script @@ -5,5 +5,5 @@ cleanup() { } trap cleanup EXIT -title "A cluster tag contradicting a fixed policy value is rejected at create\n" +title "A cluster tag contradicting a fixed policy value is rejected at create" musterr trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/jobs/cluster_policy/default_flag/script b/acceptance/bundle/resources/jobs/cluster_policy/default_flag/script index f359b41276d..fd34a51bfbc 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/default_flag/script +++ b/acceptance/bundle/resources/jobs/cluster_policy/default_flag/script @@ -8,9 +8,9 @@ trap cleanup EXIT trace $CLI bundle deploy job_id=$(read_id.py my_job) -title "defaultValue is supplied only with apply_policy_default_values (flag_on), not flag_off\n" +title "defaultValue is supplied only with apply_policy_default_values (flag_on), not flag_off" $CLI jobs get "$job_id" | jq -S '[.settings.job_clusters[] | {job_cluster_key, custom_tags: .new_cluster.custom_tags}]' -title "Neither is drift on a second plan\n" +title "Neither is drift on a second plan" trace $CLI bundle plan diff --git a/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/script b/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/script index 3c6c5f02c10..15a126e9ffd 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/script +++ b/acceptance/bundle/resources/jobs/cluster_policy/fixed_addition/script @@ -7,9 +7,9 @@ trap cleanup EXIT trace $CLI bundle deploy -title "Policy-supplied spark_version and custom_tags are not drift at any cluster-spec location\n" +title "Policy-supplied spark_version and custom_tags are not drift at any cluster-spec location" trace $CLI bundle plan -title "Each is classified remote_addition\n" +title "Each is classified remote_addition" trace $CLI bundle plan -o json | jq -S '.plan["resources.jobs.my_job"].changes | with_entries(select(.key | test("custom_tags|spark_version")))' diff --git a/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/script b/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/script index d86c89445ee..3fd3c0ca5f5 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/script +++ b/acceptance/bundle/resources/jobs/cluster_policy/owned_tag_drift/script @@ -8,14 +8,14 @@ trap cleanup EXIT trace $CLI bundle deploy job_id=$(read_id.py my_job) -title "An out-of-band change to a config-owned tag is drift, even with a policy attached\n" +title "An out-of-band change to a config-owned tag is drift, even with a policy attached" edit_resource.py jobs "$job_id" <<'EOF' for jc in r["job_clusters"]: jc["new_cluster"]["custom_tags"]["Mine"] = "changed-out-of-band" EOF trace $CLI bundle plan | contains.py "1 to change" -title "Removing a config-owned tag is a change the user asked for, not a backend addition\n" +title "Removing a config-owned tag is a change the user asked for, not a backend addition" trace update_file.py databricks.yml "Mine: mine" "Other: other" trace $CLI bundle plan -o json | jq -S '.plan["resources.jobs.my_job"].changes | with_entries(select(.key | test("custom_tags")))' diff --git a/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/script b/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/script index cdbdcaba7c0..6c5da851234 100644 --- a/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/script +++ b/acceptance/bundle/resources/jobs/cluster_policy/policy_id_scoping/script @@ -8,13 +8,13 @@ trap cleanup EXIT trace $CLI bundle deploy job_id=$(read_id.py my_job) -title "Inject a remote-only tag on the ungated cluster (no policy_id) to show it is not gated\n" +title "Inject a remote-only tag on the ungated cluster (no policy_id) to show it is not gated" edit_resource.py jobs "$job_id" <<'EOF' for jc in r["job_clusters"]: if jc["job_cluster_key"] == "ungated": jc["new_cluster"]["custom_tags"] = {"Injected": "out-of-band"} EOF -title "The gated cluster's policy tag is skipped; the ungated cluster's tag is drift\n" +title "The gated cluster's policy tag is skipped; the ungated cluster's tag is drift" trace $CLI bundle plan -o json | jq -S '.plan["resources.jobs.my_job"].changes | with_entries(select(.key | test("custom_tags")))' diff --git a/acceptance/bundle/resources/jobs/fail-on-active-runs/output.txt b/acceptance/bundle/resources/jobs/fail-on-active-runs/output.txt index 5676ea96cc6..f8abad3a9cd 100644 --- a/acceptance/bundle/resources/jobs/fail-on-active-runs/output.txt +++ b/acceptance/bundle/resources/jobs/fail-on-active-runs/output.txt @@ -9,6 +9,7 @@ Workspace: Validation OK! === deploy empty bundle + >>> [CLI] bundle deploy --fail-on-active-runs Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-jobs-active-runs-test-[UNIQUE_NAME]/default/files... Created jobs.foo @@ -16,7 +17,9 @@ Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Remove .databricks directory to simulate a fresh deployment + === deploy empty bundle again + >>> [CLI] bundle deploy --fail-on-active-runs Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-jobs-active-runs-test-[UNIQUE_NAME]/default/files... Files: 3 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt index 8421384be39..74ad57affc9 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt +++ b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt @@ -11,7 +11,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] pipelines delete [UUID] === Remove resources from config, plan and deploy + === Plan and deploy again, then summary + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged @@ -21,4 +23,5 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 0 unchanged === No delete API calls for resources that are already gone remotely + >>> print_requests.py //jobs/delete //pipelines/ --nostamp diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/output.txt b/acceptance/bundle/resources/jobs/remote_matches_config/output.txt index 18763b94677..118e6f6bd6a 100644 --- a/acceptance/bundle/resources/jobs/remote_matches_config/output.txt +++ b/acceptance/bundle/resources/jobs/remote_matches_config/output.txt @@ -12,6 +12,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Make local changes: update max_concurrent_runs to 2 + === Make remote changes via CLI: add a new tag and set timeout >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/script b/acceptance/bundle/resources/jobs/remote_matches_config/script index 3b3b62ad4a1..62455246a42 100755 --- a/acceptance/bundle/resources/jobs/remote_matches_config/script +++ b/acceptance/bundle/resources/jobs/remote_matches_config/script @@ -7,7 +7,7 @@ trace $CLI bundle plan | contains.py "0 to add, 0 to change, 0 to delete" title "Make local changes: update max_concurrent_runs to 2" update_file.py databricks.yml "max_concurrent_runs: 1" "max_concurrent_runs: 2" -title "Make remote changes via CLI: add a new tag and set timeout\n" +title "Make remote changes via CLI: add a new tag and set timeout" edit_resource.py jobs $job_id <>> print_requests.py //jobs --nostamp === Update trigger.periodic.unit and re-deploy + >>> update_file.py databricks.yml DAYS HOURS >>> [CLI] bundle plan @@ -38,6 +39,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Fetch job ID and verify remote state + >>> [CLI] jobs get [FOO_ID] { "created_time": [UNIX_TIME_MILLIS], @@ -79,6 +81,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } === Destroy the job and verify that it's removed from the state and from remote + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.foo diff --git a/acceptance/bundle/resources/jobs/update_single_node/output.txt b/acceptance/bundle/resources/jobs/update_single_node/output.txt index 21ce8c8f446..df064947479 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/output.txt +++ b/acceptance/bundle/resources/jobs/update_single_node/output.txt @@ -13,6 +13,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //jobs --nostamp === Update trigger.periodic.unit and re-deploy + >>> update_file.py databricks.yml DAYS HOURS >>> [CLI] bundle plan @@ -32,6 +33,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Fetch job ID and verify remote state + >>> [CLI] jobs get [FOO_ID] { "created_time": [UNIX_TIME_MILLIS], @@ -80,6 +82,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } === Destroy the job and verify that it's removed from the state and from remote + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.foo diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script index 64e56ce732e..7311dbad12f 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script @@ -4,7 +4,7 @@ job_id="$(read_id.py my_job)" trace $CLI bundle plan | contains.py "0 to add, 0 to change, 0 to delete" -title "Remote returns webhook destinations in a different order\n" +title "Remote returns webhook destinations in a different order" edit_resource.py jobs "$job_id" <>> [CLI] bundle summary -o json { "modified_status": "created", @@ -23,6 +24,7 @@ Files: 1 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === The create returns a long-running operation, and the created set is read back from it + >>> print_requests.py --keep //materialization-sets { "method": "POST", @@ -68,12 +70,14 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === The server assigned a set id and a member id per member + >>> [CLI] bundle summary -o json { "id": "[SET1_ID]" } === Redeploying with no config change must plan nothing + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 0 uploaded, 0 deleted @@ -82,6 +86,7 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //materialization-sets === The schedule state the server assigned must not read as drift + >>> [CLI] bundle plan -o json >>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json @@ -90,6 +95,7 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged } === The delete returns a long-running operation too, which the destroy waits on + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.features.batchfeat diff --git a/acceptance/bundle/resources/materialization_sets/batch/output.txt b/acceptance/bundle/resources/materialization_sets/batch/output.txt index a6f6b9a293b..c97c2926da3 100644 --- a/acceptance/bundle/resources/materialization_sets/batch/output.txt +++ b/acceptance/bundle/resources/materialization_sets/batch/output.txt @@ -10,6 +10,7 @@ Inserted sample data into main.matset_[UNIQUE_NAME].src_[UNIQUE_NAME] Table main.matset_[UNIQUE_NAME].src_[UNIQUE_NAME] is now visible (catalog_name=main) === A batch feature materialized offline creates, and a redeploy plans nothing + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... Created features.batchfeat @@ -28,7 +29,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Removing the member empties the set, which the update endpoint refuses in favour of the + === delete endpoint. Restoring the member plans nothing, so no refusal wrote. + >>> [CLI] bundle plan -o json >>> set_actions @@ -48,8 +51,11 @@ API message: Updating materialization set '[SET1_ID]' with no materialized featu Files: 2 uploaded, 0 deleted === Writing no value instead of an empty list is the same declaration, and has to reach the same + === endpoint: the member field emptied whole is one path rather than one per member, and the + === immutable rules match it just as readily. + >>> [CLI] bundle plan -o json >>> set_actions @@ -76,7 +82,9 @@ Files: 2 uploaded, 0 deleted } === Taking the refusal's advice: dropping the set resource rather than its members plans as a + === delete and settles. This is the deploy path's delete, not bundle destroy's. + >>> [CLI] bundle plan -o json >>> set_actions @@ -109,7 +117,9 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 1 unchanged === Repointing the destination is a recreate, not a silent no-op. The server cannot move a + === materialization in place, so the set is rebuilt at the new location. + >>> [CLI] bundle plan -o json >>> set_actions @@ -121,7 +131,9 @@ Resources: 1 created, 0 changed, 0 deleted, 1 unchanged spec:immutable old=batch_[UNIQUE_NAME]_off new=moved_[UNIQUE_NAME]_off === The schedule state is write-once, so pausing the feature is a recreate rather than the + === in-place update the endpoint would refuse + >>> [CLI] bundle plan -o json >>> set_actions diff --git a/acceptance/bundle/resources/materialization_sets/bind/output.txt b/acceptance/bundle/resources/materialization_sets/bind/output.txt index 23cfa2e1fb1..e64c51063a1 100644 --- a/acceptance/bundle/resources/materialization_sets/bind/output.txt +++ b/acceptance/bundle/resources/materialization_sets/bind/output.txt @@ -6,8 +6,11 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Unbinding leaves the set in the workspace and drops it from the bundle's state, which is + === the same shape as a set created out of band: the plan reports create, because the bundle + === has nothing recorded for it + >>> [CLI] bundle deployment unbind set1 >>> [CLI] bundle plan -o json @@ -18,13 +21,17 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Binding the existing set takes ownership of it again + >>> [CLI] bundle deployment bind set1 [UUID] --auto-approve Successfully bound materialization_set with an id '[UUID]' Run 'bundle deploy' to deploy changes to your workspace === The first plan after a bind settles. Bind seeds state from the resolved config rather + === than from a read, so the destination the read never returns is suppressed here exactly as + === it is on a set the bundle created + >>> [CLI] bundle plan -o json >>> set_plan @@ -39,6 +46,7 @@ materialized_features[member='feature_name=main.myschema.bindfeat|online_store_n materialized_features[member='feature_name=main.myschema.bindfeat|online_store_name=my_online_store|is_streaming_mode=false'].online_store_config.table_name_prefix skip input_only_not_returned === And the deploy is a no-op rather than a rebuild + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/destination/output.txt b/acceptance/bundle/resources/materialization_sets/destination/output.txt index aee99ab8f21..f31472f79ad 100644 --- a/acceptance/bundle/resources/materialization_sets/destination/output.txt +++ b/acceptance/bundle/resources/materialization_sets/destination/output.txt @@ -6,8 +6,11 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === A destination the read never returns is not drift: the config holds a value the server + === never echoes, and with no persisted value to compare it to the suppression lives in + === OverrideChangeDesc + >>> [CLI] bundle plan -o json >>> destination_changes @@ -34,7 +37,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Repointing the online destination is a real change the config asked for, so it keeps its + === persisted value and still recreates + >>> update_file.py databricks.yml catalog_name: main schema_name: myschema table_name_prefix: destfeat_on catalog_name: other @@ -69,6 +74,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml catalog_name: other catalog_name: main === The offline block is input-only whole, and repointing it recreates the same way + >>> update_file.py databricks.yml table_name_prefix: destfeat_off table_name_prefix: destfeat_moved >>> [CLI] bundle plan -o json diff --git a/acceptance/bundle/resources/materialization_sets/immutable/output.txt b/acceptance/bundle/resources/materialization_sets/immutable/output.txt index 0af82616941..3bf2ce47e7f 100644 --- a/acceptance/bundle/resources/materialization_sets/immutable/output.txt +++ b/acceptance/bundle/resources/materialization_sets/immutable/output.txt @@ -18,7 +18,9 @@ Files: 5 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === A member field the endpoint cannot alter is caught by the planner as a recreate, so the + === request never reaches the server + >>> [CLI] bundle plan -o json >>> set_action @@ -27,7 +29,9 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } === The endpoint cannot add a member either, so adding one plans a recreate of the whole set + === and warns that every member is dropped and recomputed + >>> [CLI] bundle plan -o json Warn: planning resources.materialization_sets.set1: the member for feature "main.matset_[UNIQUE_NAME].immb_[UNIQUE_NAME]" is not in the deployed set, and members cannot be added to an existing set, so the whole set will be recreated: every member is dropped and recomputed @@ -37,6 +41,7 @@ Warn: planning resources.materialization_sets.set1: the member for feature "main } === A recreate needs approval, so a deploy that cannot prompt stops before changing anything + >>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files... Warn: planning resources.materialization_sets.set1: the member for feature "main.matset_[UNIQUE_NAME].immb_[UNIQUE_NAME]" is not in the deployed set, and members cannot be added to an existing set, so the whole set will be recreated: every member is dropped and recomputed @@ -52,6 +57,7 @@ To proceed, use --auto-approve after reviewing the plan above. Files: 2 uploaded, 0 deleted === With approval the set is rebuilt holding both members, and the rebuilt set plans clean + >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files... Warn: planning resources.materialization_sets.set1: the member for feature "main.matset_[UNIQUE_NAME].immb_[UNIQUE_NAME]" is not in the deployed set, and members cannot be added to an existing set, so the whole set will be recreated: every member is dropped and recomputed @@ -72,6 +78,7 @@ Resources: 1 created, 0 changed, 1 deleted, 2 unchanged } === Removing a member is still an update + >>> [CLI] bundle plan -o json >>> set_action @@ -86,7 +93,9 @@ Files: 2 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 2 unchanged === Changing the feature name the set's materialized feature member references, from feata's to + === featb's, reads as one member removed and another added, so it plans a recreate as well + >>> [CLI] bundle plan -o json Warn: planning resources.materialization_sets.set1: the member for feature "main.matset_[UNIQUE_NAME].immb_[UNIQUE_NAME]" is not in the deployed set, and members cannot be added to an existing set, so the whole set will be recreated: every member is dropped and recomputed @@ -96,7 +105,9 @@ Warn: planning resources.materialization_sets.set1: the member for feature "main } === That recreate needs approval too, and with it the set is rebuilt with its member referencing + === featb's feature name + >>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files... Warn: planning resources.materialization_sets.set1: the member for feature "main.matset_[UNIQUE_NAME].immb_[UNIQUE_NAME]" is not in the deployed set, and members cannot be added to an existing set, so the whole set will be recreated: every member is dropped and recomputed @@ -131,7 +142,9 @@ Resources: 1 created, 0 changed, 1 deleted, 2 unchanged } === Switching the trigger from a manual cron expression to a DERIVED schedule is a recreate as + === well, and the deploy asks for approval before rebuilding the set + >>> [CLI] bundle plan -o json >>> set_action diff --git a/acceptance/bundle/resources/materialization_sets/invalid_members/output.txt b/acceptance/bundle/resources/materialization_sets/invalid_members/output.txt index 0f15a98ed33..6007a899bdf 100644 --- a/acceptance/bundle/resources/materialization_sets/invalid_members/output.txt +++ b/acceptance/bundle/resources/materialization_sets/invalid_members/output.txt @@ -1,5 +1,6 @@ === Two members differing only outside the member key are the same member declared twice + >>> musterr [CLI] bundle validate Error: Features 'main.default.feat' in materialization set 'set1' each declares more than one materialized feature with the following same fields: feature name, online store name, trigger is streaming mode, which is not allowed. at resources.materialization_sets.set1.materialized_features[0] diff --git a/acceptance/bundle/resources/materialization_sets/membership/output.txt b/acceptance/bundle/resources/materialization_sets/membership/output.txt index 23a42f20522..4a9183fd100 100644 --- a/acceptance/bundle/resources/materialization_sets/membership/output.txt +++ b/acceptance/bundle/resources/materialization_sets/membership/output.txt @@ -18,7 +18,9 @@ Files: 5 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === The cascade is per feature and runs both ways. Removing feata's online member alone is + === refused, because the delete also reaches the offline member the config still declares + >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... Error: cannot update resources.materialization_sets.set1: updating id=[UUID][0]: Materialized features '[UUID][1]' of feature 'main.matset_[UNIQUE_NAME].feata_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID][2]', which the request does not supply. Remove those as well. (400 INVALID_PARAMETER_VALUE) @@ -31,7 +33,9 @@ API message: Materialized features '[UUID][1]' of feature 'main.matset_[UNIQUE_N Files: 1 uploaded, 0 deleted === And removing feata's offline member alone is refused for the same reason, from the other + === direction + >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... Error: cannot update resources.materialization_sets.set1: updating id=[UUID][0]: Deleting materialized features '[UUID][2]' of feature 'main.matset_[UNIQUE_NAME].feata_[UNIQUE_NAME]' also deletes materialized features '[UUID][1]', which the request does not delete. Delete the unrequested materialized features as well. (400 INVALID_PARAMETER_VALUE) @@ -44,7 +48,9 @@ API message: Deleting materialized features '[UUID][2]' of feature 'main.matset_ Files: 1 uploaded, 0 deleted === Removing a whole feature's pair plans as update, not recreate, and the other feature's + === members keep their ids, so the set was reconciled rather than rebuilt + >>> [CLI] bundle plan -o json >>> set_action diff --git a/acceptance/bundle/resources/materialization_sets/mixed_triggers/output.txt b/acceptance/bundle/resources/materialization_sets/mixed_triggers/output.txt index ac4f565b58e..be1fce0fd17 100644 --- a/acceptance/bundle/resources/materialization_sets/mixed_triggers/output.txt +++ b/acceptance/bundle/resources/materialization_sets/mixed_triggers/output.txt @@ -5,9 +5,13 @@ Inserted sample data into main.myschema.src_[UNIQUE_NAME] Table main.myschema.src_[UNIQUE_NAME] is now visible (catalog_name=main) === A set cannot hold one feature that materializes in batch and one that materializes from a + === stream, because the two need different triggers. The create endpoint reserves its operation + === record before it reaches this rule, so the refusal settles that record and arrives as + === HTTP 200 rather than as an error on the call. + >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-mixed-triggers-[UNIQUE_NAME]/default/files... Created features.batchfeat diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt b/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt index 3c2dba530c2..7d71e91b070 100644 --- a/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt @@ -21,7 +21,9 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged [SET1_ID] === A feature contributing one member can have it removed alone: the cascade set is that one + === member, and the feature the request says nothing about is untouched + >>> [CLI] bundle plan -o json >>> set_action @@ -44,6 +46,7 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged ] === The update returns a long-running operation, and the reconciled set is read back from it + >>> print_requests.py --keep //materialization-sets { "method": "PATCH", diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt b/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt index 57b1b6fec7f..61a48e81f79 100644 --- a/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt @@ -1,8 +1,12 @@ === A sawtooth feature declaring only the offline and online batch pair is refused at create. + === The pair is not expanded into the third materialization: a set stamps its id on every + === member before the backend reaches that normalization, and the normalization is skipped + === whenever the id is set, so one member stays one materialized feature. + >>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... Created features.sawfeat @@ -11,6 +15,7 @@ Error: cannot create resources.materialization_sets.set1: materialization set [U Files: 7 uploaded, 0 deleted === Declaring all three creates the set + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... Created materialization_sets.set1 @@ -18,7 +23,9 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 1 unchanged === A redeploy plans nothing. Both online members carry a destination the read never returns, + === so this is also what proves the input-only suppression holds for a three-member set. + >>> [CLI] bundle plan -o json >>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json @@ -32,11 +39,14 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 2 unchanged === Every suppressed destination change, by reason + >>> jq -r .plan["resources.materialization_sets.set1"].changes | to_entries | map(select(.value.reason == "input_only_not_returned")) | length out.plan.json 7 === A sawtooth feature's three members are one cascade group, so every strict subset of them + === is refused. Removing the offline member reaches both online ones, + >>> [CLI] bundle plan -o json >>> set_action @@ -54,6 +64,7 @@ API message: Deleting materialized features '[UUID][2]' of feature 'main.myschem Files: 2 uploaded, 0 deleted === removing the online batch member names no member a delete can be applied to, + >>> [CLI] bundle plan -o json >>> set_action @@ -71,6 +82,7 @@ API message: Materialized features '[UUID][3]' of feature 'main.myschema.saw_[UN Files: 2 uploaded, 0 deleted === and removing the batch pair still leaves the streaming member behind + >>> [CLI] bundle plan -o json >>> set_action @@ -88,8 +100,11 @@ API message: Deleting materialized features '[UUID][2]', '[UUID][3]' of feature Files: 2 uploaded, 0 deleted === Removing all three satisfies the cascade but empties the set, which the update endpoint + === refuses in favour of the delete endpoint. None of the four refusals wrote anything, so + === restoring the config plans nothing. + >>> [CLI] bundle plan -o json >>> set_action diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt b/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt index abf8653d4d7..226455d7831 100644 --- a/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt @@ -8,8 +8,11 @@ Files: 3 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === A member the server cannot tear down settles the update's operation as FAILED and returns + === it with HTTP 200. The deploy fails with the operation's error code and message, naming each + === member left behind from the operation's metadata. + >>> [CLI] bundle plan -o json >>> set_action @@ -22,16 +25,22 @@ Error: cannot update resources.materialization_sets.set1: updating id=[UUID][0]: Files: 2 uploaded, 0 deleted === Nothing was written, so the removal is still pending + >>> [CLI] bundle plan -o json >>> set_action update === The delete settles FAILED the same way, and here the engine decides what the user sees: + === its destroy step forgives a delete error matching none of the cases it knows, so the + === destroy reports success and drops the state while the set stays in the workspace. The leg + === with deployment-history recording takes the one branch that records and returns the + === failure, which is why this case runs on the other leg only. + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.features.failfeat diff --git a/acceptance/bundle/resources/materialization_sets/streaming/output.txt b/acceptance/bundle/resources/materialization_sets/streaming/output.txt index b9ee84b6ae9..b4576401049 100644 --- a/acceptance/bundle/resources/materialization_sets/streaming/output.txt +++ b/acceptance/bundle/resources/materialization_sets/streaming/output.txt @@ -1,5 +1,6 @@ === A streaming member carries a trigger and no schedule, so none of the batch rules apply + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-streaming-[UNIQUE_NAME]/default/files... Created features.streamfeat @@ -9,7 +10,9 @@ Files: 4 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === A redeploy plans nothing. The schedule state the server assigns to a lone streaming member + === must not read as drift, and neither must its online destination. + >>> [CLI] bundle plan -o json >>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json @@ -23,7 +26,9 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 3 unchanged === The schedule state is write-once, so declaring one on a streaming member that had none + === plans as a recreate rather than an in-place update + >>> update_file.py databricks.yml streaming_mode: pipeline_schedule_state: PAUSED streaming_mode: @@ -35,15 +40,20 @@ Resources: 0 created, 0 changed, 0 deleted, 3 unchanged } === Repointing a member's online destination is a recreate: the server cannot move a + === materialization in place, so the whole set is rebuilt at the new location + >>> [CLI] bundle plan -o json >>> set_action recreate === Removing one feature's streaming member is an update the endpoint accepts. With no offline + === member to cascade from, each online member is its own cascade group, so the other feature's + === member is untouched and the set is left non-empty. + >>> [CLI] bundle plan -o json >>> set_action diff --git a/acceptance/bundle/resources/mcp_services/basic/output.txt b/acceptance/bundle/resources/mcp_services/basic/output.txt index 1e569e8b020..1f9b7cce46b 100644 --- a/acceptance/bundle/resources/mcp_services/basic/output.txt +++ b/acceptance/bundle/resources/mcp_services/basic/output.txt @@ -1,5 +1,6 @@ === Initial summary before deploy + >>> [CLI] bundle summary -o json { "comment": "COMMENT1", @@ -9,6 +10,7 @@ } === Verify it does not exist yet + >>> musterr [CLI] ai-gateway get-mcp-service mcp-services/main.myschema.myservice Error: Resource catalog.McpService not found: main.myschema.myservice @@ -32,6 +34,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Summary should show the id and the Catalog Explorer url + >>> [CLI] bundle summary -o json { "id": "main.myschema.myservice", @@ -39,6 +42,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Verify deployment + >>> [CLI] ai-gateway get-mcp-service mcp-services/main.myschema.myservice { "name": "mcp-services/main.myschema.myservice", @@ -46,6 +50,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Update comment (should update in place, not recreate) + >>> update_file.py databricks.yml COMMENT1 COMMENT2 >>> [CLI] bundle deploy @@ -70,6 +75,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "COMMENT2" === Change an immutable field (should plan a recreate) + >>> update_file.py databricks.yml myservice myservice-renamed >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/mcp_services/remote-delete/output.txt b/acceptance/bundle/resources/mcp_services/remote-delete/output.txt index 1abe12731aa..49250865d2f 100644 --- a/acceptance/bundle/resources/mcp_services/remote-delete/output.txt +++ b/acceptance/bundle/resources/mcp_services/remote-delete/output.txt @@ -6,9 +6,11 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Delete the resource out of band + >>> [CLI] ai-gateway delete-mcp-service mcp-services/main.myschema.myservice === Plan should detect the resource is gone and re-create it + >>> [CLI] bundle plan create mcp_services.mcp1 diff --git a/acceptance/bundle/resources/model_provider_services/basic/output.txt b/acceptance/bundle/resources/model_provider_services/basic/output.txt index 6f61bef50bb..501204ed91b 100644 --- a/acceptance/bundle/resources/model_provider_services/basic/output.txt +++ b/acceptance/bundle/resources/model_provider_services/basic/output.txt @@ -1,5 +1,6 @@ === Initial summary before deploy + >>> [CLI] bundle summary -o json { "comment": "COMMENT1", @@ -28,6 +29,7 @@ } === Verify it does not exist yet + >>> musterr [CLI] ai-gateway get-model-provider-service model-provider-services/main.myschema.myservice Error: Resource catalog.ModelProviderService not found: main.myschema.myservice @@ -70,6 +72,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Summary should show the id and the Catalog Explorer url + >>> [CLI] bundle summary -o json { "id": "main.myschema.myservice", @@ -77,6 +80,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Verify deployment + >>> [CLI] ai-gateway get-model-provider-service model-provider-services/main.myschema.myservice { "name": "model-provider-services/main.myschema.myservice", @@ -84,6 +88,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Update comment (should update in place, not recreate) + >>> update_file.py databricks.yml COMMENT1 COMMENT2 >>> [CLI] bundle deploy @@ -127,6 +132,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "COMMENT2" === Change an immutable field (should plan a recreate) + >>> update_file.py databricks.yml EXTERNAL_MODEL_PROVIDER_TYPE_CUSTOM EXTERNAL_MODEL_PROVIDER_TYPE_ANTHROPIC >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/model_provider_services/remote-delete/output.txt b/acceptance/bundle/resources/model_provider_services/remote-delete/output.txt index 49b01ec7128..8268fdc8a2c 100644 --- a/acceptance/bundle/resources/model_provider_services/remote-delete/output.txt +++ b/acceptance/bundle/resources/model_provider_services/remote-delete/output.txt @@ -6,9 +6,11 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Delete the resource out of band + >>> [CLI] ai-gateway delete-model-provider-service model-provider-services/main.myschema.myservice === Plan should detect the resource is gone and re-create it + >>> [CLI] bundle plan create model_provider_services.mps1 diff --git a/acceptance/bundle/resources/model_services/basic/output.txt b/acceptance/bundle/resources/model_services/basic/output.txt index 42d7686eee0..24e7de6dc29 100644 --- a/acceptance/bundle/resources/model_services/basic/output.txt +++ b/acceptance/bundle/resources/model_services/basic/output.txt @@ -1,5 +1,6 @@ === Initial summary before deploy + >>> [CLI] bundle summary -o json { "comment": "COMMENT1", @@ -9,6 +10,7 @@ } === Verify it does not exist yet + >>> musterr [CLI] ai-gateway get-model-service model-services/main.myschema.myservice Error: Resource catalog.ModelService not found: main.myschema.myservice @@ -32,6 +34,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Summary should show the id and the Catalog Explorer url + >>> [CLI] bundle summary -o json { "id": "main.myschema.myservice", @@ -39,6 +42,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Verify deployment + >>> [CLI] ai-gateway get-model-service model-services/main.myschema.myservice { "name": "model-services/main.myschema.myservice", @@ -46,6 +50,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Update comment (should update in place, not recreate) + >>> update_file.py databricks.yml COMMENT1 COMMENT2 >>> [CLI] bundle deploy @@ -70,6 +75,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "COMMENT2" === Change an immutable field (should plan a recreate) + >>> update_file.py databricks.yml myservice myservice-renamed >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/model_services/remote-delete/output.txt b/acceptance/bundle/resources/model_services/remote-delete/output.txt index 7cedf83e463..c98e07b6438 100644 --- a/acceptance/bundle/resources/model_services/remote-delete/output.txt +++ b/acceptance/bundle/resources/model_services/remote-delete/output.txt @@ -6,9 +6,11 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Delete the resource out of band + >>> [CLI] ai-gateway delete-model-service model-services/main.myschema.myservice === Plan should detect the resource is gone and re-create it + >>> [CLI] bundle plan create model_services.ms1 diff --git a/acceptance/bundle/resources/model_serving_endpoints/basic/script b/acceptance/bundle/resources/model_serving_endpoints/basic/script index c12cb1bd8f2..d5baf65e9cd 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/basic/script +++ b/acceptance/bundle/resources/model_serving_endpoints/basic/script @@ -28,7 +28,7 @@ get_output=$($CLI serving-endpoints get $ENDPOINT_NAME) endpoint_id=$(echo "$get_output" | jq -r '.id') add_repl "$endpoint_id" ENDPOINT_ID_1 -title "Print the GET endpoint details\n" +title "Print the GET endpoint details" echo "$get_output" | jq '{name, creator}' export ENDPOINT_NAME="test-endpoint-$UNIQUE_NAME-2" @@ -44,5 +44,5 @@ get_output=$($CLI serving-endpoints get $ENDPOINT_NAME) endpoint_id=$(echo "$get_output" | jq -r '.id') add_repl "$endpoint_id" ENDPOINT_ID_2 -title "Print the GET endpoint details\n" +title "Print the GET endpoint details" echo "$get_output" | jq '{name, creator}' diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/output.txt index 6c9c004b74e..f38c9337157 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-mse-recreated-same-name-[UNIQUE_NAME]/default/files... Created model_serving_endpoints.my_endpoint @@ -14,6 +15,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Delete and recreate remotely with the same name + >>> [CLI] serving-endpoints delete mse-endpoint-[UNIQUE_NAME] >>> [CLI] serving-endpoints create mse-endpoint-[UNIQUE_NAME] --no-wait @@ -31,6 +33,7 @@ Original endpoint id: [ORIGINAL_ENDPOINT_ID] Remote recreated endpoint id: [REMOTE_RECREATED_ENDPOINT_ID] === Plan after out-of-band recreate + >>> [CLI] bundle plan update model_serving_endpoints.my_endpoint.permissions @@ -49,6 +52,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } === After deploy the permissions are recorded under the new endpoint id, so the plan is clean + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt index 4aec6358c5c..e7636b6f498 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/output.txt @@ -1,5 +1,6 @@ === Deploy without telemetry_config + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created model_serving_endpoints.endpoint1 @@ -7,7 +8,9 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Configure telemetry out of band + === Remote-only telemetry_config is not drift + >>> [CLI] bundle plan --output json { "telemetry_config": { @@ -20,6 +23,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Deploy does not remove it + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 0 uploaded, 0 deleted @@ -31,5 +35,6 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged } === Only the out-of-band telemetry PATCH was sent + >>> print_requests.py --method PATCH --oneline //serving-endpoints {"method": "PATCH", "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", "body": {"telemetry_config": {"table_names": {"logs_table": "main.default.other_logs"}}}} diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt index 093d5875e98..89ef95a1d88 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created model_serving_endpoints.endpoint1 @@ -7,6 +8,7 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Update config and telemetry_config + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Updated model_serving_endpoints.endpoint1 @@ -14,10 +16,12 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Plan converges + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Request order: config PUT, poll, telemetry PATCH + >>> print_requests.py --get --oneline //serving-endpoints 1 {"method": "POST", "path": "/api/2.0/serving-endpoints", "body": {"config": {"served_entities": [{"entity_name": "main.default.test_model", "entity_version": "1", "name": "prod", "scale_to_zero_enabled": true, "workload_size": "Small"}]}, "name": "test-endpoint", "telemetry_config": {"inference_table_config": {"sampling_fraction": 0.5}, "table_names": {"logs_table": "main.default.test_logs"}}}} 2 {"method": "GET", "path": "/api/2.0/serving-endpoints/test-endpoint"} @@ -27,6 +31,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged 2 {"method": "GET", "path": "/api/2.0/serving-endpoints/test-endpoint"} === Remove telemetry_config + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Updated model_serving_endpoints.endpoint1 @@ -37,9 +42,11 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged null === Plan converges after removal + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Removal sends an empty telemetry PATCH + >>> print_requests.py --method PATCH --oneline //serving-endpoints {"method": "PATCH", "path": "/api/2.0/serving-endpoints/test-endpoint/telemetry-config", "body": {}} diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/output.txt index c0f856ff6c5..effca618046 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created model_serving_endpoints.endpoint1 @@ -7,10 +8,12 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Plan is a no-op despite write-only secret and burst_scaling_enabled not round-tripping + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The write-only fields are skipped as input_only (confirms the matched rules) + >>> [CLI] bundle plan --output json { "config.served_entities[0].burst_scaling_enabled": { @@ -47,6 +50,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } === Redeploy is a no-op (no config/ai-gateway/tags update calls) + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 0 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/output.txt b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/output.txt index 41d5cd758a7..d3f739d3c0f 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/output.txt @@ -34,6 +34,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === verify there's no persistent drift + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/output.txt b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/output.txt index a8d5f8628be..4284f085c73 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/output.txt @@ -23,6 +23,7 @@ Files: 2 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === There a bug in TF where it does not actually update the email notifications for a serving endpoint. + >>> print_requests.py //serving-endpoints >>> [CLI] serving-endpoints get [ENDPOINT_ID] diff --git a/acceptance/bundle/resources/models/basic/output.txt b/acceptance/bundle/resources/models/basic/output.txt index 6727e0d9d9e..1a728254936 100644 --- a/acceptance/bundle/resources/models/basic/output.txt +++ b/acceptance/bundle/resources/models/basic/output.txt @@ -4,6 +4,7 @@ >>> export MODEL_DESCRIPTION=original-description-[UNIQUE_NAME] === create a model + >>> [CLI] bundle plan create models.my_model @@ -26,6 +27,7 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged >>> export MODEL_DESCRIPTION=new-description-[UNIQUE_NAME] === update the description, this should update the description remotely as well + >>> [CLI] bundle plan update models.my_model @@ -48,6 +50,7 @@ Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged >>> export MODEL_NAME=new-name-[UNIQUE_NAME] === update the name, this should recreate the model with the new name + >>> [CLI] bundle plan recreate models.my_model @@ -68,6 +71,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged } === add a new tag, this should be a no-op; terraform does make an update request but since update request has no tags in it, it does nothing; direct skips request completely + >>> [CLI] bundle plan >>> [CLI] bundle deploy -qq diff --git a/acceptance/bundle/resources/models/readplan-permissions/output.txt b/acceptance/bundle/resources/models/readplan-permissions/output.txt index a4e059b61a7..d60c6904ac4 100644 --- a/acceptance/bundle/resources/models/readplan-permissions/output.txt +++ b/acceptance/bundle/resources/models/readplan-permissions/output.txt @@ -1,5 +1,6 @@ === Initial deploy: model created with permissions + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created models.mymodel @@ -29,7 +30,8 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } -=== Remove manager permission and description, deploy from planUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +=== Remove manager permission and description, deploy from plan +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Updated models.mymodel Updated models.mymodel.permissions Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/permissions/factcheck/script b/acceptance/bundle/resources/permissions/factcheck/script index cbddce58087..7045272200d 100644 --- a/acceptance/bundle/resources/permissions/factcheck/script +++ b/acceptance/bundle/resources/permissions/factcheck/script @@ -16,7 +16,7 @@ add_repl "$sql_warehouse_id" SQL_WAREHOUSE_ID user=deco-test-user@databricks.com -title "Permissions API simply takes the latest level in the request for a given principal\n" +title "Permissions API simply takes the latest level in the request for a given principal" python3 check_permissions.py $user jobs $job_id CAN_VIEW,CAN_MANAGE_RUN CAN_MANAGE_RUN python3 check_permissions.py $user jobs $job_id CAN_MANAGE_RUN,CAN_VIEW CAN_VIEW python3 check_permissions.py $user jobs $job_id CAN_VIEW,CAN_MANAGE_RUN,CAN_MANAGE CAN_MANAGE @@ -24,7 +24,7 @@ python3 check_permissions.py $user jobs $job_id CAN_MANAGE,CAN_MANAGE_RUN CAN_MA python3 check_permissions.py $CURRENT_USER_NAME jobs $job_id CAN_MANAGE,IS_OWNER IS_OWNER -title "Since we take the most recent level, IS_OWNER is lost, which results in the error due to lack of owner defined\n" +title "Since we take the most recent level, IS_OWNER is lost, which results in the error due to lack of owner defined" musterr python3 check_permissions.py $CURRENT_USER_NAME jobs $job_id IS_OWNER,CAN_MANAGE CAN_MANAGE python3 check_permissions.py $user sql/warehouses $sql_warehouse_id CAN_VIEW,CAN_USE CAN_USE diff --git a/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/output.txt b/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/output.txt index b7972be4f1b..aee6bfa7887 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/output.txt +++ b/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/output.txt @@ -7,6 +7,7 @@ Files: 1 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Permissions are set on the live space + >>> [CLI] api get /api/2.0/permissions/genie/[GENIE_SPACE_ID] { "object_type": "genie-space", @@ -34,9 +35,11 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Trash the space out-of-band + >>> [CLI] genie trash-space [GENIE_SPACE_ID] === Permissions API still returns the ACLs for the trashed space (no 404) + >>> [CLI] api get /api/2.0/permissions/genie/[GENIE_SPACE_ID] { "object_type": "genie-space", @@ -64,6 +67,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Plan recreates the gone space + >>> [CLI] bundle plan create genie_spaces.foo update genie_spaces.foo.permissions diff --git a/acceptance/bundle/resources/permissions/jobs/added_remotely/output.txt b/acceptance/bundle/resources/permissions/jobs/added_remotely/output.txt index 81b09d6fcb2..a0a6fdf9c65 100644 --- a/acceptance/bundle/resources/permissions/jobs/added_remotely/output.txt +++ b/acceptance/bundle/resources/permissions/jobs/added_remotely/output.txt @@ -56,6 +56,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged } === Add permissions out of band + >>> [CLI] permissions set jobs [JOB_WITH_PERMISSIONS_ID] --json @remote_add.json { "access_control_list": [ diff --git a/acceptance/bundle/resources/permissions/jobs/delete_one/script b/acceptance/bundle/resources/permissions/jobs/delete_one/script index ef6e2142e07..738a6622064 100644 --- a/acceptance/bundle/resources/permissions/jobs/delete_one/script +++ b/acceptance/bundle/resources/permissions/jobs/delete_one/script @@ -26,7 +26,7 @@ job_id="$(read_id.py job_with_permissions)" $CLI permissions get jobs "$job_id" | gron.py --sort-arrays access_control_list | grep -v display_name > out.permissions_create.txt rm -f "$OUT_REQUESTS" -title "Delete one permission and deploy again\n" +title "Delete one permission and deploy again" grep -v DELETE databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace $CLI bundle deploy print_requests.py --nostamp //jobs/ | gron.py --sort-arrays access_control_list > out.requests_update.txt diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/output.txt b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/output.txt index 3dbeecb21c7..b7692d6da2f 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/output.txt +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/output.txt @@ -60,6 +60,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Delete permissions remotely + >>> [CLI] permissions set jobs [JOB_WITH_PERMISSIONS_ID] --json @remote_delete.json { "access_control_list": [ diff --git a/acceptance/bundle/resources/permissions/jobs/update/script b/acceptance/bundle/resources/permissions/jobs/update/script index 0396e6b68e4..0807e13e8cf 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/script +++ b/acceptance/bundle/resources/permissions/jobs/update/script @@ -11,28 +11,28 @@ job_id="$(read_id.py job_with_permissions)" trace $CLI permissions get jobs "$job_id" rm -f "$OUT_REQUESTS" -title "Update one permission and deploy again\n" +title "Update one permission and deploy again" update_file.py databricks.yml CAN_VIEW CAN_MANAGE trace $CLI bundle plan -o json | nostamp > out.plan_update.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_update.json trace $CLI bundle plan -title "Delete one permission and deploy again\n" +title "Delete one permission and deploy again" grep -v DELETE_ONE databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace $CLI bundle plan -o json | nostamp > out.plan_delete_one.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_delete_one.json trace $CLI bundle plan -title "Delete the whole block and deploy again\n" +title "Delete the whole block and deploy again" grep -v PERMISSIONS databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace $CLI bundle plan -o json | nostamp > out.plan_delete_all.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_delete_all.json trace $CLI bundle plan -title "Restore original config\n" +title "Restore original config" trace diff.py databricks.yml.saved databricks.yml mv databricks.yml.saved databricks.yml @@ -41,7 +41,7 @@ trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_restore_original.json trace $CLI bundle plan -title "Set permissions: []\n" +title "Set permissions: []" grep -vE '(PERMISSIONS|DELETE_ONE)' databricks.yml > tmp.yml && mv tmp.yml databricks.yml update_file.py databricks.yml '# permissions: [] # EXPLICIT_EMPTY' 'permissions: [] # EXPLICIT_EMPTY' trace $CLI bundle plan -o json | nostamp > out.plan_set_empty.json diff --git a/acceptance/bundle/resources/permissions/pipelines/recreate/output.txt b/acceptance/bundle/resources/permissions/pipelines/recreate/output.txt index 09eb1af7f79..fe98cda6bc1 100644 --- a/acceptance/bundle/resources/permissions/pipelines/recreate/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/recreate/output.txt @@ -1,5 +1,6 @@ === Deploy a pipeline with permissions + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created pipelines.foo @@ -30,6 +31,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged pipelines foo.permissions id='/pipelines/[OLD_PIPELINE_ID]' === Switch from catalog to storage: the pipeline is recreated with a new ID + >>> update_file.py databricks.yml catalog: main storage: dbfs:/my-storage >>> update_file.py databricks.yml target: ${resources.schemas.bar.name} #target: ${resources.schemas.bar.name} @@ -46,6 +48,7 @@ pipelines foo.permissions id='/pipelines/[OLD_PIPELINE_ID]' } === Deploy recreates the pipeline and applies the permissions to the new one + >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... @@ -78,6 +81,7 @@ Resources: 1 created, 1 changed, 1 deleted, 1 unchanged } === The permissions are recorded under the new ID, so the next plan is a no-op + >>> read_state.py pipelines foo.permissions id pipelines foo.permissions id='/pipelines/[NEW_PIPELINE_ID]' diff --git a/acceptance/bundle/resources/permissions/pipelines/update/script b/acceptance/bundle/resources/permissions/pipelines/update/script index 3be2be37147..209ab959dcb 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/script +++ b/acceptance/bundle/resources/permissions/pipelines/update/script @@ -9,21 +9,21 @@ pipeline_id="$(read_id.py foo)" trace $CLI permissions get pipelines "$pipeline_id" rm -f "$OUT_REQUESTS" -title "Update one permission and deploy again\n" +title "Update one permission and deploy again" update_file.py databricks.yml CAN_VIEW CAN_MANAGE trace $CLI bundle plan -o json | nostamp > out.plan_update.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_update.json trace $CLI bundle plan -title "Delete one permission and deploy again\n" +title "Delete one permission and deploy again" grep -v DELETE_ONE databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace $CLI bundle plan -o json | nostamp > out.plan_delete_one.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_delete_one.json trace $CLI bundle plan -title "Delete the whole block and deploy again\n" +title "Delete the whole block and deploy again" grep -v PERMISSIONS databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace cat databricks.yml trace $CLI bundle plan -o json | nostamp > out.plan_delete_all.json @@ -31,7 +31,7 @@ trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' --sort > out.requests_delete_all.json trace $CLI bundle plan -title "Restore original config\n" +title "Restore original config" mv databricks.yml.saved databricks.yml trace $CLI bundle plan -o json | nostamp > out.plan_restore.json trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/pipelines/auto-approve/output.txt b/acceptance/bundle/resources/pipelines/auto-approve/output.txt index 3795c17db35..5f920d3d2fa 100644 --- a/acceptance/bundle/resources/pipelines/auto-approve/output.txt +++ b/acceptance/bundle/resources/pipelines/auto-approve/output.txt @@ -7,6 +7,7 @@ Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Assert the pipeline is created + >>> [CLI] pipelines get [UUID] { "spec": { @@ -29,11 +30,13 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } -=== Assert the job is created{ +=== Assert the job is created +{ "name": "test-bundle-job-[UNIQUE_NAME]" } === Remove resources from configuration. + >>> rm resources.yml >>> [CLI] bundle plan @@ -43,6 +46,7 @@ delete pipelines.bar Plan: 0 to add, 0 to change, 2 to delete, 0 unchanged === Try to redeploy the bundle - should fail without --auto-approve + >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... diff --git a/acceptance/bundle/resources/pipelines/cluster_policy/script b/acceptance/bundle/resources/pipelines/cluster_policy/script index dfe1ad7caa2..de3678d8ec0 100644 --- a/acceptance/bundle/resources/pipelines/cluster_policy/script +++ b/acceptance/bundle/resources/pipelines/cluster_policy/script @@ -7,5 +7,5 @@ trap cleanup EXIT trace $CLI bundle deploy -title "A pipeline cluster with policy_id converges: the Pipelines API does not expand the policy\n" +title "A pipeline cluster with policy_id converges: the Pipelines API does not expand the policy" trace $CLI bundle plan diff --git a/acceptance/bundle/resources/pipelines/drift/parameters/output.txt b/acceptance/bundle/resources/pipelines/drift/parameters/output.txt index 46711138b98..833234d0d22 100644 --- a/acceptance/bundle/resources/pipelines/drift/parameters/output.txt +++ b/acceptance/bundle/resources/pipelines/drift/parameters/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-parameters/default/files... Created pipelines.my @@ -7,10 +8,12 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Plan is a no-op: pipeline parameters round-trip on read + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Redeploy is a no-op (no update call) + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-parameters/default/files... Files: 0 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt index 831e9e36a7d..22e76c3ff57 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt @@ -105,6 +105,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged } === Fetch pipeline ID and verify remote state + >>> [CLI] pipelines get [MY_ID_2] { "creator_user_name": "[USERNAME]", @@ -141,10 +142,12 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged } === Verify that original pipeline is gone + >>> musterr [CLI] pipelines get [MY_ID] Error: The specified pipeline [MY_ID] was not found. === Destroy the pipeline and verify that it's removed from the state and from remote + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.pipelines.my diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt index 4152f08a59b..ab4731f3771 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt @@ -95,6 +95,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged } === Fetch pipeline ID and verify remote state + >>> [CLI] pipelines get [MY_ID_2] { "creator_user_name": "[USERNAME]", @@ -125,10 +126,12 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged } === Verify that original pipeline is gone + >>> musterr [CLI] pipelines get [MY_ID] Error: The specified pipeline [MY_ID] was not found. === Destroy the pipeline and verify that it's removed from the state and from remote + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.pipelines.my diff --git a/acceptance/bundle/resources/pipelines/recreate/output.txt b/acceptance/bundle/resources/pipelines/recreate/output.txt index 3ee69c5c6c1..e9ec01b878e 100644 --- a/acceptance/bundle/resources/pipelines/recreate/output.txt +++ b/acceptance/bundle/resources/pipelines/recreate/output.txt @@ -10,7 +10,8 @@ Created schemas.bar Files: 3 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged -=== Assert the pipeline is created with catalogWarning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes. +=== Assert the pipeline is created with catalog +Warning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes. at resources.pipelines.foo.development in databricks.yml:14:20 @@ -40,6 +41,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Switch from catalog to storage (triggers recreation) + >>> update_file.py databricks.yml catalog: main storage: dbfs:/my-storage >>> update_file.py databricks.yml target: ${resources.schemas.bar.name} #target: ${resources.schemas.bar.name} @@ -54,6 +56,7 @@ recreate pipelines.foo Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged === Redeploy the bundle (should recreate the pipeline) + >>> errcode [CLI] bundle deploy --force-lock Warning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes. at resources.pipelines.foo.development diff --git a/acceptance/bundle/resources/pipelines/remote_matches_config/output.txt b/acceptance/bundle/resources/pipelines/remote_matches_config/output.txt index 774b5582008..3c706fb19dd 100644 --- a/acceptance/bundle/resources/pipelines/remote_matches_config/output.txt +++ b/acceptance/bundle/resources/pipelines/remote_matches_config/output.txt @@ -6,7 +6,9 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Make local change: run_as user -> changed@example.test + === Force remote run_as to match the new config value + >>> [CLI] bundle plan update pipelines.my_pipeline diff --git a/acceptance/bundle/resources/pipelines/update/output.txt b/acceptance/bundle/resources/pipelines/update/output.txt index f84d08b7557..6e9ba5d28ba 100644 --- a/acceptance/bundle/resources/pipelines/update/output.txt +++ b/acceptance/bundle/resources/pipelines/update/output.txt @@ -37,6 +37,7 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Fetch pipeline ID and verify remote state + >>> [CLI] pipelines get [MY_ID] { "creator_user_name": "[USERNAME]", @@ -67,6 +68,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Destroy the pipeline and verify that it's removed from the state and from remote + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.pipelines.my diff --git a/acceptance/bundle/resources/postgres_branches/purge_on_delete/output.txt b/acceptance/bundle/resources/postgres_branches/purge_on_delete/output.txt index b38d10cd744..ab4490ab49c 100644 --- a/acceptance/bundle/resources/postgres_branches/purge_on_delete/output.txt +++ b/acceptance/bundle/resources/postgres_branches/purge_on_delete/output.txt @@ -55,4 +55,5 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ === bundle destroy + >>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ diff --git a/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/output.txt b/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/output.txt index 5441f61b43b..6df440d7033 100644 --- a/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/output.txt +++ b/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/output.txt @@ -1,5 +1,6 @@ === Step 1: deploy with purge_on_delete unset + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-branch-purge-transitions-[UNIQUE_NAME]/default/files... Created postgres_branches.branch @@ -11,6 +12,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged (unset) === Step 2: set purge_on_delete: true + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-branch-purge-transitions-[UNIQUE_NAME]/default/files... Updated postgres_branches.branch @@ -21,6 +23,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged json.state.resources.postgres_branches.branch.state.purge_on_delete = true; === Step 3: flip to purge_on_delete: false + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-branch-purge-transitions-[UNIQUE_NAME]/default/files... Updated postgres_branches.branch @@ -31,6 +34,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged json.state.resources.postgres_branches.branch.state.purge_on_delete = false; === Step 4: remove the line again + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-branch-purge-transitions-[UNIQUE_NAME]/default/files... Updated postgres_branches.branch @@ -41,4 +45,5 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged (unset) === bundle destroy (branch DELETE must be a plain DELETE, no ?purge=true) + >>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ diff --git a/acceptance/bundle/resources/postgres_branches/recreate/output.txt b/acceptance/bundle/resources/postgres_branches/recreate/output.txt index 78cf8d611dc..2ff2198bb26 100644 --- a/acceptance/bundle/resources/postgres_branches/recreate/output.txt +++ b/acceptance/bundle/resources/postgres_branches/recreate/output.txt @@ -38,6 +38,7 @@ Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Change branch_id (should trigger recreation in direct, update in terraform) + >>> cat databricks.yml bundle: name: deploy-postgres-branch-recreate-[UNIQUE_NAME] @@ -74,7 +75,9 @@ Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === Fetch new branch ID + === Destroy and verify cleanup + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.postgres_branches.main diff --git a/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/output.txt b/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/output.txt index 76b348aa3fc..cd51e2ca2f8 100644 --- a/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/output.txt +++ b/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-postgres-branch-fork-[UNIQUE_NAME]/default/files... Created postgres_branches.dev_branch @@ -8,6 +9,7 @@ Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Change the fork point: source_branch_time is immutable, so this recreates + >>> update_file.py databricks.yml [FORK_TIME_BEFORE] [FORK_TIME_AFTER] >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/postgres_branches/replace_existing/output.txt b/acceptance/bundle/resources/postgres_branches/replace_existing/output.txt index 5f8d6a0823f..25b1d5aa650 100644 --- a/acceptance/bundle/resources/postgres_branches/replace_existing/output.txt +++ b/acceptance/bundle/resources/postgres_branches/replace_existing/output.txt @@ -33,4 +33,5 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ === bundle destroy + >>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ diff --git a/acceptance/bundle/resources/postgres_branches/update_expire_time/output.txt b/acceptance/bundle/resources/postgres_branches/update_expire_time/output.txt index a87b943addc..8519e1e90a7 100644 --- a/acceptance/bundle/resources/postgres_branches/update_expire_time/output.txt +++ b/acceptance/bundle/resources/postgres_branches/update_expire_time/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-postgres-branch-expire-[UNIQUE_NAME]/default/files... Created postgres_branches.dev_branch @@ -27,6 +28,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Change expire_time and re-deploy + >>> update_file.py databricks.yml [EXPIRE_TIME_BEFORE] [EXPIRE_TIME_AFTER] >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/output.txt b/acceptance/bundle/resources/postgres_branches/update_protected/output.txt index 1a2eadd2c68..9ac72b133b5 100644 --- a/acceptance/bundle/resources/postgres_branches/update_protected/output.txt +++ b/acceptance/bundle/resources/postgres_branches/update_protected/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle validate Name: update-postgres-branch-[UNIQUE_NAME] Target: default @@ -45,6 +46,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Verify no_change (no changes) + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -58,6 +60,7 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ === Update is_protected to true and re-deploy + >>> update_file.py databricks.yml is_protected: false is_protected: true >>> cat databricks.yml @@ -120,6 +123,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } === Restore is_protected to false + >>> update_file.py databricks.yml is_protected: true is_protected: false >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/postgres_branches/update_ttl/output.txt b/acceptance/bundle/resources/postgres_branches/update_ttl/output.txt index a0f75d30074..fb31167f31e 100644 --- a/acceptance/bundle/resources/postgres_branches/update_ttl/output.txt +++ b/acceptance/bundle/resources/postgres_branches/update_ttl/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-postgres-branch-ttl-[UNIQUE_NAME]/default/files... Created postgres_branches.dev_branch @@ -8,6 +9,7 @@ Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Change ttl and re-deploy + >>> update_file.py databricks.yml ttl: 604800s ttl: 259200s >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/postgres_data_apis/basic/output.txt b/acceptance/bundle/resources/postgres_data_apis/basic/output.txt index bba81e51742..a8ee6145b38 100644 --- a/acceptance/bundle/resources/postgres_data_apis/basic/output.txt +++ b/acceptance/bundle/resources/postgres_data_apis/basic/output.txt @@ -41,6 +41,7 @@ create postgres_roles.owner Plan: 5 to add, 0 to change, 0 to delete, 0 unchanged === Deploy enables the Data API + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-pg-data-api-[UNIQUE_NAME]/default/files... Created postgres_branches.production @@ -90,10 +91,12 @@ Resources: Name: projects/test-pg-proj-[UNIQUE_NAME]/branches/production/roles/app-owner === Plan after deploy shows no changes + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged === Removing the resource turns off the Data API but keeps the database + >>> [CLI] bundle plan delete postgres_data_apis.my_data_api diff --git a/acceptance/bundle/resources/postgres_data_apis/out-of-band/output.txt b/acceptance/bundle/resources/postgres_data_apis/out-of-band/output.txt index 23fb9af2d78..ccaa335d2af 100644 --- a/acceptance/bundle/resources/postgres_data_apis/out-of-band/output.txt +++ b/acceptance/bundle/resources/postgres_data_apis/out-of-band/output.txt @@ -1,5 +1,6 @@ === Deploy enables the Data API + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/oob-pg-data-api-[UNIQUE_NAME]/default/files... Created postgres_branches.production @@ -26,6 +27,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged } === Change the spec out of band + >>> [CLI] postgres update-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api spec --json {"spec": {"db_schemas": ["public"], "db_max_rows": 7}} { "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api", @@ -42,6 +44,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged } === Plan detects the change from status, and deploy restores the config + >>> [CLI] bundle plan update postgres_data_apis.my_data_api @@ -72,9 +75,11 @@ Resources: 0 created, 1 changed, 0 deleted, 4 unchanged Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged === Disable the Data API out of band + >>> [CLI] postgres delete-data-api projects/test-pg-proj-[UNIQUE_NAME]/branches/production/databases/my-database/data-api === Plan detects the Data API is gone and re-creates it + >>> [CLI] bundle plan create postgres_data_apis.my_data_api diff --git a/acceptance/bundle/resources/postgres_data_apis/parent_only/output.txt b/acceptance/bundle/resources/postgres_data_apis/parent_only/output.txt index a53475e579b..67331b2a5f0 100644 --- a/acceptance/bundle/resources/postgres_data_apis/parent_only/output.txt +++ b/acceptance/bundle/resources/postgres_data_apis/parent_only/output.txt @@ -1,5 +1,6 @@ === Validate rejects a Data API that sets only parent + >>> musterr [CLI] bundle validate Error: postgres_data_apis.my_data_api: at least one Data API setting besides parent is required, for example db_schemas at resources.postgres_data_apis.my_data_api @@ -14,6 +15,7 @@ Workspace: Found 1 error === Deploy fails the same way before anything is created + >>> musterr [CLI] bundle deploy Error: postgres_data_apis.my_data_api: at least one Data API setting besides parent is required, for example db_schemas at resources.postgres_data_apis.my_data_api diff --git a/acceptance/bundle/resources/postgres_data_apis/recreate/output.txt b/acceptance/bundle/resources/postgres_data_apis/recreate/output.txt index 1f6b735af21..65346bbb2df 100644 --- a/acceptance/bundle/resources/postgres_data_apis/recreate/output.txt +++ b/acceptance/bundle/resources/postgres_data_apis/recreate/output.txt @@ -1,5 +1,6 @@ === Deploy the Data API on the first database + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-pg-data-api-[UNIQUE_NAME]/default/files... Created postgres_branches.production @@ -27,6 +28,7 @@ Resources: 6 created, 0 changed, 0 deleted, 0 unchanged } === Moving the Data API to another database recreates it + >>> update_file.py databricks.yml postgres_databases.first.name postgres_databases.second.name >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/postgres_data_apis/update/output.txt b/acceptance/bundle/resources/postgres_data_apis/update/output.txt index 125829cd631..0b0e758cea9 100644 --- a/acceptance/bundle/resources/postgres_data_apis/update/output.txt +++ b/acceptance/bundle/resources/postgres_data_apis/update/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-pg-data-api-[UNIQUE_NAME]/default/files... Created postgres_branches.production @@ -11,6 +12,7 @@ Files: 0 uploaded, 0 deleted Resources: 5 created, 0 changed, 0 deleted, 0 unchanged === Change a spec field + >>> update_file.py databricks.yml db_max_rows: 100 db_max_rows: 500 >>> [CLI] bundle plan @@ -42,6 +44,7 @@ Resources: 0 created, 1 changed, 0 deleted, 4 unchanged } === Remove a spec field: update_mask=spec keeps a field the request omits, so it keeps its last value + >>> [CLI] bundle plan update postgres_data_apis.my_data_api @@ -69,6 +72,7 @@ Resources: 0 created, 1 changed, 0 deleted, 4 unchanged } === Plan after update shows no changes + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged diff --git a/acceptance/bundle/resources/postgres_data_apis/url_reference/output.txt b/acceptance/bundle/resources/postgres_data_apis/url_reference/output.txt index 9645a0d4566..05549b12d5b 100644 --- a/acceptance/bundle/resources/postgres_data_apis/url_reference/output.txt +++ b/acceptance/bundle/resources/postgres_data_apis/url_reference/output.txt @@ -1,5 +1,6 @@ === The first deploy resolves status.url into the job parameter + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/url-ref-pg-data-api-[UNIQUE_NAME]/default/files... Created jobs.my_job @@ -25,6 +26,7 @@ Resources: 6 created, 0 changed, 0 deleted, 0 unchanged ] === An in-place update of the Data API does not update the job that references status.url + >>> update_file.py databricks.yml db_max_rows: 100 db_max_rows: 500 >>> [CLI] bundle plan @@ -47,6 +49,7 @@ Resources: 0 created, 1 changed, 0 deleted, 5 unchanged ] === Plan after update shows no changes + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 6 unchanged diff --git a/acceptance/bundle/resources/postgres_databases/recreate/output.txt b/acceptance/bundle/resources/postgres_databases/recreate/output.txt index 2619af86a8b..fef88268bd2 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate/output.txt +++ b/acceptance/bundle/resources/postgres_databases/recreate/output.txt @@ -143,6 +143,7 @@ recreate postgres_databases.my_database Plan: 1 to add, 0 to change, 1 to delete, 3 unchanged === Recreate requires approval: non-interactive deploy must abort, not silently delete the database + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-database-recreate-[UNIQUE_NAME]/default/files... @@ -185,6 +186,7 @@ Resources: 1 created, 0 changed, 1 deleted, 3 unchanged } === Fetch database and verify it exists after recreation + >>> [CLI] postgres get-database [MY_DATABASE_ID]-v2 { "database_id": "test-database-[UNIQUE_NAME]-v2", @@ -198,6 +200,7 @@ Resources: 1 created, 0 changed, 1 deleted, 3 unchanged } === Destroy and verify cleanup + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.postgres_branches.main diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/output.txt b/acceptance/bundle/resources/postgres_databases/replace_existing/output.txt index 944668621bf..ec23011d880 100644 --- a/acceptance/bundle/resources/postgres_databases/replace_existing/output.txt +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/output.txt @@ -1,5 +1,6 @@ === Deploy project, branch, role, and database + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/replace-existing-database-[UNIQUE_NAME]/default/files... Created postgres_branches.main @@ -10,15 +11,18 @@ Files: 0 uploaded, 0 deleted Resources: 4 created, 0 changed, 0 deleted, 0 unchanged === Unbind the database: it now exists on the branch but is no longer tracked by the bundle + >>> [CLI] bundle deployment unbind my_database === Plan after unbind: the bundle wants to (re-)create the database + >>> [CLI] bundle plan create postgres_databases.my_database Plan: 1 to add, 0 to change, 0 to delete, 3 unchanged === Re-deploy: replace_existing adopts the already-existing database instead of erroring + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/replace-existing-database-[UNIQUE_NAME]/default/files... Created postgres_databases.my_database @@ -26,6 +30,7 @@ Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 3 unchanged === Confirm the database is under management again + >>> [CLI] postgres get-database projects/test-pg-proj-[UNIQUE_NAME]/branches/main/databases/my-database { "database_id": "my-database", diff --git a/acceptance/bundle/resources/postgres_databases/update/output.txt b/acceptance/bundle/resources/postgres_databases/update/output.txt index b94e0180d7e..2a81331e9a4 100644 --- a/acceptance/bundle/resources/postgres_databases/update/output.txt +++ b/acceptance/bundle/resources/postgres_databases/update/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle validate Name: update-postgres-database-[UNIQUE_NAME] Target: default @@ -43,6 +44,7 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged } === Verify no changes + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged @@ -56,6 +58,7 @@ Resources: 0 created, 0 changed, 0 deleted, 4 unchanged >>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ === Update postgres_database and re-deploy + >>> update_file.py databricks.yml postgres_database: initial_db_name postgres_database: renamed_db_name >>> cat databricks.yml @@ -120,6 +123,7 @@ Resources: 0 created, 1 changed, 0 deleted, 3 unchanged } === Restore postgres_database to original value + >>> update_file.py databricks.yml postgres_database: renamed_db_name postgres_database: initial_db_name >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/postgres_endpoints/add_settings/output.txt b/acceptance/bundle/resources/postgres_endpoints/add_settings/output.txt index 64e5b4d2d1d..3e185589de1 100644 --- a/acceptance/bundle/resources/postgres_endpoints/add_settings/output.txt +++ b/acceptance/bundle/resources/postgres_endpoints/add_settings/output.txt @@ -1,5 +1,6 @@ === Initial deployment with no settings block + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created postgres_branches.main @@ -9,6 +10,7 @@ Files: 0 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Add a settings block and re-deploy + >>> [CLI] bundle plan update postgres_endpoints.my_endpoint @@ -23,6 +25,7 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged >>> print_requests.py --del-body project_id,branch_id,endpoint_id --sort //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ === Plan again: the addition must leave no drift behind + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged diff --git a/acceptance/bundle/resources/postgres_endpoints/recreate/output.txt b/acceptance/bundle/resources/postgres_endpoints/recreate/output.txt index 7ff6430ff42..634046b4244 100644 --- a/acceptance/bundle/resources/postgres_endpoints/recreate/output.txt +++ b/acceptance/bundle/resources/postgres_endpoints/recreate/output.txt @@ -159,10 +159,12 @@ Resources: 1 created, 0 changed, 1 deleted, 2 unchanged } === Fetch endpoint and verify it exists after recreation + >>> [CLI] postgres get-endpoint [MY_ENDPOINT_ID]-v2 "ENDPOINT_TYPE_READ_ONLY" === Destroy and verify cleanup + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.postgres_branches.main diff --git a/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/output.txt b/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/output.txt index 09186727f10..59954a2cd43 100644 --- a/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/output.txt +++ b/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/output.txt @@ -1,5 +1,6 @@ === Initial deployment with suspend_timeout_duration: 300s + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/remove-postgres-endpoint-suspend-[UNIQUE_NAME]/default/files... Created postgres_branches.main @@ -24,6 +25,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } === Remove suspend_timeout_duration: neither engine can apply it + >>> update_file.py databricks.yml suspend_timeout_duration: 300s >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/postgres_endpoints/replace_existing/output.txt b/acceptance/bundle/resources/postgres_endpoints/replace_existing/output.txt index 61ae1fbba07..be609b6da13 100644 --- a/acceptance/bundle/resources/postgres_endpoints/replace_existing/output.txt +++ b/acceptance/bundle/resources/postgres_endpoints/replace_existing/output.txt @@ -34,4 +34,5 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ === bundle destroy + >>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/output.txt b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/output.txt index 1a613bbb5b8..7e3ead200f7 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/output.txt +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle validate Name: update-postgres-endpoint-[UNIQUE_NAME] Target: default @@ -44,6 +45,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } === Verify no changes + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged @@ -57,6 +59,7 @@ Resources: 0 created, 0 changed, 0 deleted, 3 unchanged >>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ === Update endpoint autoscaling and re-deploy + >>> update_file.py databricks.yml autoscaling_limit_max_cu: 8 autoscaling_limit_max_cu: 4 >>> cat databricks.yml @@ -120,6 +123,7 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged } === Restore endpoint autoscaling to original value + >>> update_file.py databricks.yml autoscaling_limit_max_cu: 4 autoscaling_limit_max_cu: 8 >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/output.txt b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/output.txt index 8bc4e2561b9..5dd75731258 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/output.txt +++ b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/output.txt @@ -1,5 +1,6 @@ === Initial deployment with pg_settings.statement_timeout: 8000 + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created postgres_branches.main @@ -9,6 +10,7 @@ Files: 0 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Change the value of one pg_settings key + >>> update_file.py databricks.yml statement_timeout: 8000 statement_timeout: 16000 >>> [CLI] bundle plan @@ -27,6 +29,7 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged >>> print_requests.py --del-body project_id,branch_id,endpoint_id --sort //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ === Plan again: the change must leave no drift behind + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged diff --git a/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/output.txt b/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/output.txt index 68c0c87cbbb..6bde76314ec 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/output.txt +++ b/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/output.txt @@ -1,5 +1,6 @@ === Initial deployment with suspend_timeout_duration: 300s + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-postgres-endpoint-suspend-[UNIQUE_NAME]/default/files... Created postgres_branches.main @@ -24,9 +25,11 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } === Change suspend_timeout_duration to 600s + >>> update_file.py databricks.yml suspend_timeout_duration: 300s suspend_timeout_duration: 600s === Plan and deploy + >>> [CLI] bundle plan update postgres_endpoints.my_endpoint diff --git a/acceptance/bundle/resources/postgres_projects/add_default_endpoint_settings/output.txt b/acceptance/bundle/resources/postgres_projects/add_default_endpoint_settings/output.txt index a7947234202..e7c8300fd41 100644 --- a/acceptance/bundle/resources/postgres_projects/add_default_endpoint_settings/output.txt +++ b/acceptance/bundle/resources/postgres_projects/add_default_endpoint_settings/output.txt @@ -1,5 +1,6 @@ === Initial deployment with no default_endpoint_settings block + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created postgres_projects.my_project @@ -7,6 +8,7 @@ Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Add the whole default_endpoint_settings block and re-deploy + >>> update_file.py databricks.yml #TO_ADD: >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/postgres_projects/factcheck/update_mask_spec/output.txt b/acceptance/bundle/resources/postgres_projects/factcheck/update_mask_spec/output.txt index d4ac98d0a77..32ecc0ddd3d 100644 --- a/acceptance/bundle/resources/postgres_projects/factcheck/update_mask_spec/output.txt +++ b/acceptance/bundle/resources/postgres_projects/factcheck/update_mask_spec/output.txt @@ -1,5 +1,6 @@ === Create accepts a minimal body; the server materializes the rest + >>> [CLI] postgres get-project projects/factcheck-spec-[UNIQUE_NAME] { "name": "projects/factcheck-spec-[UNIQUE_NAME]", @@ -24,6 +25,7 @@ } === update_mask=spec with a partial body (omits history_retention_duration + default_endpoint_settings) + >>> [CLI] postgres update-project projects/factcheck-spec-[UNIQUE_NAME] spec --json {"spec":{"display_name":"changed","pg_version":16}} { "name": "projects/factcheck-spec-[UNIQUE_NAME]", @@ -63,6 +65,7 @@ } === Set default_endpoint_settings to a non-default value (leaf mask) + >>> [CLI] postgres get-project projects/factcheck-spec-[UNIQUE_NAME] { "name": "projects/factcheck-spec-[UNIQUE_NAME]", @@ -105,6 +108,7 @@ } === update_mask=spec with a body that now omits the non-default default_endpoint_settings + >>> [CLI] postgres update-project projects/factcheck-spec-[UNIQUE_NAME] spec --json {"spec":{"display_name":"changed2","pg_version":16,"history_retention_duration":"604800s"}} { "name": "projects/factcheck-spec-[UNIQUE_NAME]", diff --git a/acceptance/bundle/resources/postgres_projects/factcheck/update_mask_star/output.txt b/acceptance/bundle/resources/postgres_projects/factcheck/update_mask_star/output.txt index e968375589b..c9348d093b1 100644 --- a/acceptance/bundle/resources/postgres_projects/factcheck/update_mask_star/output.txt +++ b/acceptance/bundle/resources/postgres_projects/factcheck/update_mask_star/output.txt @@ -1,5 +1,6 @@ === Create accepts a minimal body; the server materializes the rest + >>> [CLI] postgres get-project projects/factcheck-star-[UNIQUE_NAME] { "name": "projects/factcheck-star-[UNIQUE_NAME]", @@ -24,6 +25,7 @@ } === update_mask=* with a partial body (omits history_retention_duration + default_endpoint_settings) + >>> [CLI] postgres update-project projects/factcheck-star-[UNIQUE_NAME] * --json {"spec":{"display_name":"changed","pg_version":16}} Error: Field '[SPEC_FIELD]' is in update_mask but not provided in request @@ -45,6 +47,7 @@ Exit code: 1 } === Set default_endpoint_settings to a non-default value (leaf mask) + >>> [CLI] postgres get-project projects/factcheck-star-[UNIQUE_NAME] { "name": "projects/factcheck-star-[UNIQUE_NAME]", @@ -87,6 +90,7 @@ Exit code: 1 } === update_mask=* with a body that now omits the non-default default_endpoint_settings + >>> [CLI] postgres update-project projects/factcheck-star-[UNIQUE_NAME] * --json {"spec":{"display_name":"changed2","pg_version":16,"history_retention_duration":"604800s"}} Error: Field '[SPEC_FIELD]' is in update_mask but not provided in request diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete/output.txt b/acceptance/bundle/resources/postgres_projects/purge_on_delete/output.txt index f910036be1e..7e69ce597c5 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete/output.txt +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete/output.txt @@ -64,4 +64,5 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ === bundle destroy + >>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/output.txt b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/output.txt index f4d53b0ce18..47c29ab1213 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/output.txt +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/output.txt @@ -1,5 +1,6 @@ === Step 1: deploy with purge_on_delete unset + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-purge-transitions-[UNIQUE_NAME]/default/files... Created postgres_projects.proj @@ -10,6 +11,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged (unset) === Step 2: set purge_on_delete: true + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-purge-transitions-[UNIQUE_NAME]/default/files... Updated postgres_projects.proj @@ -20,6 +22,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged json.state.resources.postgres_projects.proj.state.purge_on_delete = true; === Step 3: flip to purge_on_delete: false + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-purge-transitions-[UNIQUE_NAME]/default/files... Updated postgres_projects.proj @@ -30,6 +33,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged json.state.resources.postgres_projects.proj.state.purge_on_delete = false; === Step 4: remove the line again + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-purge-transitions-[UNIQUE_NAME]/default/files... Updated postgres_projects.proj @@ -40,4 +44,5 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged (unset) === bundle destroy (must be a plain DELETE, no ?purge=true) + >>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ diff --git a/acceptance/bundle/resources/postgres_projects/recreate/output.txt b/acceptance/bundle/resources/postgres_projects/recreate/output.txt index eb9d70a8a15..7a9b4177f08 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate/output.txt +++ b/acceptance/bundle/resources/postgres_projects/recreate/output.txt @@ -30,6 +30,7 @@ Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Change project_id (should trigger recreation in direct, update in terraform) + >>> cat databricks.yml bundle: name: deploy-postgres-recreate-[UNIQUE_NAME] @@ -60,7 +61,9 @@ Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 0 unchanged === Fetch new project ID + === Destroy and verify cleanup + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.postgres_projects.my_project diff --git a/acceptance/bundle/resources/postgres_projects/recreate/script b/acceptance/bundle/resources/postgres_projects/recreate/script index 3d539655db2..6e792e6dfbe 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate/script +++ b/acceptance/bundle/resources/postgres_projects/recreate/script @@ -46,7 +46,6 @@ trace $CLI bundle destroy --auto-approve print_requests.py --del-body parent,project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.txt title "Old project is gone after recreate" -echo # Local testserver returns not found; the cloud soft-deletes and returns the project with delete_time set. errcode $CLI postgres get-project $project_id_1 &> old_project.tmp grep -q -e 'not found' -e '"delete_time"' old_project.tmp && echo "Old project is deleted" || echo "Old project still exists" diff --git a/acceptance/bundle/resources/postgres_projects/remove_history_retention/output.txt b/acceptance/bundle/resources/postgres_projects/remove_history_retention/output.txt index ec3dd1a62e4..76bc2a78205 100644 --- a/acceptance/bundle/resources/postgres_projects/remove_history_retention/output.txt +++ b/acceptance/bundle/resources/postgres_projects/remove_history_retention/output.txt @@ -1,5 +1,6 @@ === Initial deployment with history_retention_duration: 604800s + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/remove-postgres-project-retention-[UNIQUE_NAME]/default/files... Created postgres_projects.my_project @@ -30,6 +31,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Remove history_retention_duration and re-deploy + >>> update_file.py databricks.yml history_retention_duration: 604800s >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/postgres_projects/update_default_endpoint_autoscaling/output.txt b/acceptance/bundle/resources/postgres_projects/update_default_endpoint_autoscaling/output.txt index 8863b8bc2c0..5e9334bcba9 100644 --- a/acceptance/bundle/resources/postgres_projects/update_default_endpoint_autoscaling/output.txt +++ b/acceptance/bundle/resources/postgres_projects/update_default_endpoint_autoscaling/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created postgres_projects.my_project @@ -7,6 +8,7 @@ Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Change one field inside default_endpoint_settings + >>> update_file.py databricks.yml autoscaling_limit_max_cu: 4 autoscaling_limit_max_cu: 8 >>> [CLI] bundle plan @@ -30,6 +32,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Plan again: the change must leave no drift behind + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/output.txt b/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/output.txt index e074e12a5e7..a12981ce49a 100644 --- a/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/output.txt +++ b/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-postgres-project-default-suspend-[UNIQUE_NAME]/default/files... Created postgres_projects.my_project @@ -30,6 +31,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Change default_endpoint_settings.suspend_timeout_duration and re-deploy + >>> update_file.py databricks.yml suspend_timeout_duration: 300s suspend_timeout_duration: 600s >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/output.txt b/acceptance/bundle/resources/postgres_projects/update_display_name/output.txt index 874ecf991fb..d538708636d 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/output.txt +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle validate Name: update-postgres-project-[UNIQUE_NAME] Target: default @@ -48,6 +49,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Verify no changes + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -61,6 +63,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged >>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ === Update display_name and re-deploy + >>> update_file.py databricks.yml Original Name Updated Name >>> cat databricks.yml @@ -121,6 +124,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Restore display_name to original value + >>> update_file.py databricks.yml Updated Name Original Name >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/postgres_projects/update_history_retention/output.txt b/acceptance/bundle/resources/postgres_projects/update_history_retention/output.txt index 71fbe88c3b9..0383e390053 100644 --- a/acceptance/bundle/resources/postgres_projects/update_history_retention/output.txt +++ b/acceptance/bundle/resources/postgres_projects/update_history_retention/output.txt @@ -1,5 +1,6 @@ === Initial deployment with history_retention_duration: 604800s + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-postgres-project-retention-[UNIQUE_NAME]/default/files... Created postgres_projects.my_project @@ -30,6 +31,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Change history_retention_duration to 259200s and re-deploy + >>> update_file.py databricks.yml history_retention_duration: 604800s history_retention_duration: 259200s >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/postgres_projects/update_keeps_children/output.txt b/acceptance/bundle/resources/postgres_projects/update_keeps_children/output.txt index 37bb70a4cfd..b4246687ef5 100644 --- a/acceptance/bundle/resources/postgres_projects/update_keeps_children/output.txt +++ b/acceptance/bundle/resources/postgres_projects/update_keeps_children/output.txt @@ -1,5 +1,6 @@ === Initial deployment (project + branch + role) + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-keeps-children-[UNIQUE_NAME]/default/files... Created postgres_branches.my_branch @@ -9,6 +10,7 @@ Files: 0 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === In-place project update must not recreate the branch or role + >>> update_file.py databricks.yml Original Name Updated Name >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/output.txt b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/output.txt index f3f4318a653..5b5f30999bd 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/output.txt +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/output.txt @@ -1,5 +1,6 @@ === Deploy project, branch, and role + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/inherited-role-bind-[UNIQUE_NAME]/default/files... Created postgres_branches.main @@ -9,14 +10,17 @@ Files: 0 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Unbind the role: it now exists on the branch but is untracked + >>> [CLI] bundle deployment unbind my_role === Bind the existing role to take explicit ownership of it + >>> [CLI] bundle deployment bind my_role projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role --auto-approve Successfully bound postgres_role with an id 'projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role' Run 'bundle deploy' to deploy changes to your workspace === Re-deploy now succeeds: the role is managed, not re-created + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/inherited-role-bind-[UNIQUE_NAME]/default/files... Files: 0 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/output.txt b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/output.txt index f6f94cb232d..a306ddf23e3 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/output.txt +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/output.txt @@ -1,5 +1,6 @@ === Deploy project, branch, and role + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/inherited-role-conflict-[UNIQUE_NAME]/default/files... Created postgres_branches.main @@ -9,15 +10,18 @@ Files: 0 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Unbind the role: it now exists on the branch but is no longer tracked by the bundle + >>> [CLI] bundle deployment unbind my_role === Plan after unbind: the bundle wants to (re-)create the role + >>> [CLI] bundle plan create postgres_roles.my_role Plan: 1 to add, 0 to change, 0 to delete, 2 unchanged === Re-deploy: creating the already-existing role fails + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/inherited-role-conflict-[UNIQUE_NAME]/default/files... Error: cannot create resources.postgres_roles.my_role: role with that name already exists (400 BAD_REQUEST) diff --git a/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/output.txt b/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/output.txt index 8743a283141..a22699e9b4b 100644 --- a/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/output.txt +++ b/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/output.txt @@ -138,6 +138,7 @@ Resources: 1 created, 0 changed, 1 deleted, 2 unchanged } === Fetch role and verify the new postgres_role value is live + >>> [CLI] postgres get-role [MY_ROLE_ID] { "name": "[MY_ROLE_ID]", @@ -157,6 +158,7 @@ Resources: 1 created, 0 changed, 1 deleted, 2 unchanged } === Destroy and verify cleanup + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.postgres_branches.main diff --git a/acceptance/bundle/resources/postgres_roles/recreate/output.txt b/acceptance/bundle/resources/postgres_roles/recreate/output.txt index dc0c7e73f6d..7d4882124b9 100644 --- a/acceptance/bundle/resources/postgres_roles/recreate/output.txt +++ b/acceptance/bundle/resources/postgres_roles/recreate/output.txt @@ -138,6 +138,7 @@ Resources: 1 created, 0 changed, 1 deleted, 2 unchanged } === Fetch role and verify it exists after recreation + >>> [CLI] postgres get-role [MY_ROLE_ID]-v2 { "name": "[MY_ROLE_ID]-v2", @@ -157,6 +158,7 @@ Resources: 1 created, 0 changed, 1 deleted, 2 unchanged } === Destroy and verify cleanup + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.postgres_branches.main diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/output.txt b/acceptance/bundle/resources/postgres_roles/replace_existing/output.txt index 4073ac96ab0..1b558424d80 100644 --- a/acceptance/bundle/resources/postgres_roles/replace_existing/output.txt +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/output.txt @@ -1,5 +1,6 @@ === Deploy project, branch, and role + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/replace-existing-role-[UNIQUE_NAME]/default/files... Created postgres_branches.main @@ -9,15 +10,18 @@ Files: 0 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Unbind the role: it now exists on the branch but is no longer tracked by the bundle + >>> [CLI] bundle deployment unbind my_role === Plan after unbind: the bundle wants to (re-)create the role + >>> [CLI] bundle plan create postgres_roles.my_role Plan: 1 to add, 0 to change, 0 to delete, 2 unchanged === Re-deploy: replace_existing adopts the already-existing role instead of erroring + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/replace-existing-role-[UNIQUE_NAME]/default/files... Created postgres_roles.my_role @@ -25,6 +29,7 @@ Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 2 unchanged === Confirm the role is under management again + >>> [CLI] postgres get-role projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role { "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/roles/test-role", diff --git a/acceptance/bundle/resources/postgres_roles/update/output.txt b/acceptance/bundle/resources/postgres_roles/update/output.txt index f95c6c0118e..5af34ddd97f 100644 --- a/acceptance/bundle/resources/postgres_roles/update/output.txt +++ b/acceptance/bundle/resources/postgres_roles/update/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle validate Name: update-postgres-role-[UNIQUE_NAME] Target: default @@ -47,6 +48,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } === Verify no changes + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged @@ -60,6 +62,7 @@ Resources: 0 created, 0 changed, 0 deleted, 3 unchanged >>> print_requests.py --del-body project_id,branch_id,endpoint_id,database_id,role_id,catalog_id,synced_table_id --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ === Toggle attributes.createdb and re-deploy + >>> update_file.py databricks.yml createdb: false createdb: true >>> cat databricks.yml @@ -125,6 +128,7 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged } === Restore attributes.createdb to original value + >>> update_file.py databricks.yml createdb: true createdb: false >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/postgres_snapshot_schedules/basic/output.txt b/acceptance/bundle/resources/postgres_snapshot_schedules/basic/output.txt index c0ecf60045f..c9c9a91f29a 100644 --- a/acceptance/bundle/resources/postgres_snapshot_schedules/basic/output.txt +++ b/acceptance/bundle/resources/postgres_snapshot_schedules/basic/output.txt @@ -1,5 +1,6 @@ === Create with a snapshot schedule + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-pg-ss-basic-[UNIQUE_NAME]/default/files... Created postgres_projects.my_project @@ -21,6 +22,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Update the schedule cadence + >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-pg-ss-basic-[UNIQUE_NAME]/default/files... Updated postgres_snapshot_schedules.main_schedule @@ -41,6 +43,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } === Remove the snapshot schedule resource (disables automatic snapshots) + >>> cat databricks.yml bundle: name: deploy-pg-ss-basic-[UNIQUE_NAME] @@ -69,6 +72,7 @@ Resources: 0 created, 0 changed, 1 deleted, 1 unchanged } === Plan again to confirm no drift after removal + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/postgres_snapshot_schedules/out-of-band/output.txt b/acceptance/bundle/resources/postgres_snapshot_schedules/out-of-band/output.txt index e11ac76f602..1d47a0d4c65 100644 --- a/acceptance/bundle/resources/postgres_snapshot_schedules/out-of-band/output.txt +++ b/acceptance/bundle/resources/postgres_snapshot_schedules/out-of-band/output.txt @@ -1,5 +1,6 @@ === Deploy with a snapshot schedule + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-pg-ss-oob-[UNIQUE_NAME]/default/files... Created postgres_projects.my_project @@ -21,6 +22,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Change the schedule out of band via a direct API call + >>> [CLI] postgres update-snapshot-schedule projects/test-pg-proj-[UNIQUE_NAME]/branches/production/snapshot-schedule schedule --json {"schedule":[{"daily_schedule":{"hour":9},"retention":"604800s"}]} { "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production/snapshot-schedule", @@ -35,12 +37,14 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Plan detects the drift + >>> [CLI] bundle plan update postgres_snapshot_schedules.main_schedule Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged === Deploy reconciles the schedule back to the configured cadence + >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-pg-ss-oob-[UNIQUE_NAME]/default/files... Updated postgres_snapshot_schedules.main_schedule diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_capped/output.txt b/acceptance/bundle/resources/postgres_synced_tables/recreate_capped/output.txt index 0ba65f672e9..e870338d02e 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate_capped/output.txt +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate_capped/output.txt @@ -9,6 +9,7 @@ Files: 0 uploaded, 0 deleted Resources: 4 created, 0 changed, 0 deleted, 0 unchanged === The delete never completes; the capped wait times out and the recreate fails + >>> DATABRICKS_BUNDLE_RESOURCE_MAX_WAIT=1 errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-capped-postgres-synced-table-[UNIQUE_NAME]/default/files... Error: cannot recreate resources.postgres_synced_tables.my_table: waiting after deleting id=synced_tables/lakebase_test_[UNIQUE_NAME].public.trips_synced: timed out: synced table still exists, waiting for deletion to complete diff --git a/acceptance/bundle/resources/registered_models/aliases/output.txt b/acceptance/bundle/resources/registered_models/aliases/output.txt index 9fbb886d90a..1286ee381df 100644 --- a/acceptance/bundle/resources/registered_models/aliases/output.txt +++ b/acceptance/bundle/resources/registered_models/aliases/output.txt @@ -1,5 +1,6 @@ === Deploy a model that declares no aliases + >>> print_requests.py //unity-catalog/models { "method": "POST", @@ -12,6 +13,7 @@ } === Out of band: a training run registers two versions and points 'champion' at version 1 + >>> register_model_version main.default.model-aliases-[UNIQUE_NAME] 1 @@ -28,6 +30,7 @@ } === No aliases section: 'champion' is skipped as unmanaged, not drift + >>> plan_aliases { "aliases": { @@ -50,6 +53,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Declare aliases: 'champion' moves to version 2 and 'challenger' points at version 1 + >>> plan_aliases { "aliases": { @@ -148,6 +152,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ] === Redeploy is a no-op: the fields the backend fills in on each alias are skipped as output_only + >>> plan_aliases { "aliases[alias_name='challenger'].catalog_name": { @@ -196,6 +201,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Drift: out of band, 'champion' moves back to version 1 and 'challenger' is deleted; deploy restores both + >>> print_requests.py //unity-catalog/models { "method": "PUT", @@ -289,6 +295,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ] === Drop 'challenger' from the config: it is deleted, and so is 'staging', set out of band meanwhile + >>> print_requests.py //unity-catalog/models { "method": "PUT", @@ -377,6 +384,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ] === An empty aliases list keeps the aliases managed and deletes them all + >>> plan_aliases { "aliases[alias_name='champion']": { @@ -416,6 +424,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged [] === Remove the aliases section: aliases are unmanaged again, so one set out of band stays + >>> print_requests.py //unity-catalog/models { "method": "PUT", diff --git a/acceptance/bundle/resources/registered_models/aliases_converge/output.txt b/acceptance/bundle/resources/registered_models/aliases_converge/output.txt index 11e41cc81c1..8b030f89a60 100644 --- a/acceptance/bundle/resources/registered_models/aliases_converge/output.txt +++ b/acceptance/bundle/resources/registered_models/aliases_converge/output.txt @@ -2,6 +2,7 @@ >>> export COMMENT=original comment === Initial deployment with aliases set: the model is created, but has no version 1 for the alias yet + >>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-registered-models-aliases-[UNIQUE_NAME]/default/files... Error: cannot create resources.registered_models.my_registered_model: waiting after creating id=main.default.my-registered-model-aliases-[UNIQUE_NAME]: Model version 1 of registered model main.default.my-registered-model-aliases-[UNIQUE_NAME] does not exist. (404 RESOURCE_DOES_NOT_EXIST) @@ -14,12 +15,14 @@ API message: Model version 1 of registered model main.default.my-registered-mode Files: 2 uploaded, 0 deleted === The model is tracked: the next plan retries the alias as an update instead of creating the model again + >>> [CLI] bundle plan update registered_models.my_registered_model Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged === A training run registers version 1, and the deploy sets the alias + >>> register_model_version main.default.my-registered-model-aliases-[UNIQUE_NAME] 1 @@ -38,10 +41,12 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged ] === Plan is a no-op: the fields the backend fills in on the alias do not drift + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Edit the comment: an in-place update, not a recreate + >>> [CLI] bundle plan update registered_models.my_registered_model @@ -54,6 +59,7 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Re-plan after the edit converges: no further changes + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/registered_models/basic/output.txt b/acceptance/bundle/resources/registered_models/basic/output.txt index 0cfe7b53077..fb05660caf4 100644 --- a/acceptance/bundle/resources/registered_models/basic/output.txt +++ b/acceptance/bundle/resources/registered_models/basic/output.txt @@ -8,6 +8,7 @@ >>> export SCHEMA_NAME=default === create catalog and schema to test diff functionality + >>> [CLI] catalogs create mycatalog-[UNIQUE_NAME] { "full_name": "mycatalog-[UNIQUE_NAME]" @@ -19,6 +20,7 @@ } === create the registered model + >>> [CLI] bundle plan create registered_models.my_registered_model @@ -39,6 +41,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === update the comment, this should not recreate + >>> [CLI] bundle plan update registered_models.my_registered_model @@ -59,6 +62,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === update the name, this should recreate + >>> [CLI] bundle plan recreate registered_models.my_registered_model @@ -79,6 +83,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged } === update the catalog name, this should recreate + >>> [CLI] bundle plan recreate registered_models.my_registered_model @@ -99,6 +104,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged } === update the schema name, this should recreate + >>> [CLI] bundle plan recreate registered_models.my_registered_model diff --git a/acceptance/bundle/resources/registered_models/comment_cleared/output.txt b/acceptance/bundle/resources/registered_models/comment_cleared/output.txt index 76ef93f002f..89b0f6aff6c 100644 --- a/acceptance/bundle/resources/registered_models/comment_cleared/output.txt +++ b/acceptance/bundle/resources/registered_models/comment_cleared/output.txt @@ -1,5 +1,6 @@ === Deploy with a comment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created registered_models.model1 @@ -8,6 +9,7 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Remove the comment from the configuration and redeploy + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated registered_models.model1 @@ -15,14 +17,17 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged === The comment the bundle no longer declares + >>> [CLI] api get /api/2.1/unity-catalog/models/main.test-schema-[UNIQUE_NAME].test-model-[UNIQUE_NAME] "" === Redeploy + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === The UpdateRegisteredModel payloads the deploys sent + >>> print_requests.py --method PATCH //unity-catalog/models { "method": "PATCH", diff --git a/acceptance/bundle/resources/registered_models/drift/browse_only/output.txt b/acceptance/bundle/resources/registered_models/drift/browse_only/output.txt index 33aa6f879e5..93e5689b1c7 100644 --- a/acceptance/bundle/resources/registered_models/drift/browse_only/output.txt +++ b/acceptance/bundle/resources/registered_models/drift/browse_only/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created registered_models.model1 @@ -7,10 +8,12 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Plan is a no-op despite the backend computing browse_only + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The remote-only browse_only is skipped as output_only (confirms the matched rule) + >>> [CLI] bundle plan --output json { "action": "skip", @@ -19,6 +22,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } === Redeploy is a no-op (no UpdateRegisteredModel call) + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 0 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/schemas/auto-approve/output.txt b/acceptance/bundle/resources/schemas/auto-approve/output.txt index 1b520137437..36ad7132ba5 100644 --- a/acceptance/bundle/resources/schemas/auto-approve/output.txt +++ b/acceptance/bundle/resources/schemas/auto-approve/output.txt @@ -11,13 +11,15 @@ Files: 5 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Assert the schema is created + >>> [CLI] schemas get main.test-schema-[UNIQUE_NAME] { "full_name": "main.test-schema-[UNIQUE_NAME]", "comment": "This schema was created from DABs" } -=== Assert the pipeline is created and uses the schemaWarning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes. +=== Assert the pipeline is created and uses the schema +Warning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes. at resources.pipelines.foo.development in databricks.yml:14:20 @@ -49,6 +51,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Create a volume in the schema, and add a file to it. This ensures that the schema has some data in it and deletion will fail unless the generated terraform configuration has force_destroy set to true. + >>> [CLI] volumes create main test-schema-[UNIQUE_NAME] test-volume-[UNIQUE_NAME] MANAGED { "full_name": "main.test-schema-[UNIQUE_NAME].test-volume-[UNIQUE_NAME]" @@ -58,9 +61,11 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged test-file-[UNIQUE_NAME].txt -> dbfs:/Volumes/main/test-schema-[UNIQUE_NAME]/test-volume-[UNIQUE_NAME]/test-file-[UNIQUE_NAME].txt === Remove the UC schema from the resource configuration. + >>> rm schema.yml === Try to redeploy the bundle - should fail without --auto-approve + >>> [CLI] bundle deploy Warning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes. at resources.pipelines.foo.development @@ -77,6 +82,7 @@ To proceed, use --auto-approve after reviewing the plan above. Files: 1 uploaded, 1 deleted === Test cleanup + >>> [CLI] bundle destroy --auto-approve Warning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes. at resources.pipelines.foo.development @@ -98,4 +104,5 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 2 deleted === Assert the schema is deleted + Exit code: 1 diff --git a/acceptance/bundle/resources/schemas/comment_out_of_band/output.txt b/acceptance/bundle/resources/schemas/comment_out_of_band/output.txt index ff470fc8acd..7bcd7ba6aad 100644 --- a/acceptance/bundle/resources/schemas/comment_out_of_band/output.txt +++ b/acceptance/bundle/resources/schemas/comment_out_of_band/output.txt @@ -1,5 +1,6 @@ === Deploy a schema that does not declare a comment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -7,7 +8,9 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Set the comment out of band, the way Catalog Explorer does + === The remote comment is drift, so the plan updates the schema + >>> [CLI] bundle plan --output json { "comment": { @@ -17,6 +20,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Redeploy clears the comment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated schemas.schema1 @@ -24,16 +28,19 @@ Files: 0 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === The comment is gone + >>> [CLI] api get /api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME] "" === Redeploy is a no-op + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged === The UpdateSchema payloads the deploys sent + >>> print_requests.py --method PATCH //unity-catalog/schemas { "method": "PATCH", diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt b/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt index 66b4711e1d6..f490e82091b 100644 --- a/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created schemas.schema1 @@ -7,10 +8,12 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Plan is a no-op despite UC auto-populating managed properties + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === The remote-only properties map is skipped as a backend default (confirms the matched rule) + >>> [CLI] bundle plan --output json { "properties": { @@ -29,6 +32,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } === Redeploy is a no-op (no UpdateSchema call) + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Files: 0 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/schemas/recreate/output.txt b/acceptance/bundle/resources/schemas/recreate/output.txt index 8c11c4e0fe4..a9fdc5e0d19 100644 --- a/acceptance/bundle/resources/schemas/recreate/output.txt +++ b/acceptance/bundle/resources/schemas/recreate/output.txt @@ -28,6 +28,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Update catalog_name and re-deploy + >>> update_file.py databricks.yml catalog_name: main catalog_name: newmain >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/schemas/update/output.txt b/acceptance/bundle/resources/schemas/update/output.txt index ab18f7e09c7..0a8d72ff849 100644 --- a/acceptance/bundle/resources/schemas/update/output.txt +++ b/acceptance/bundle/resources/schemas/update/output.txt @@ -18,6 +18,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged schemas schema1 id='main.myschema' name='myschema' catalog_name='main' comment='COMMENT1' === Update comment and re-deploy + >>> update_file.py databricks.yml COMMENT1 COMMENT2 >>> [CLI] bundle deploy @@ -33,6 +34,7 @@ json.body.comment = "COMMENT2"; schemas schema1 id='main.myschema' name='myschema' catalog_name='main' comment='COMMENT2' === Restore comment to original value and re-deploy + >>> update_file.py databricks.yml COMMENT2 COMMENT1 >>> [CLI] bundle deploy diff --git a/acceptance/bundle/resources/secret_scopes/basic/output.txt b/acceptance/bundle/resources/secret_scopes/basic/output.txt index c432f2996de..0b392361159 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/output.txt +++ b/acceptance/bundle/resources/secret_scopes/basic/output.txt @@ -1,5 +1,6 @@ === create the secret scope + >>> [CLI] bundle plan -o json >>> [CLI] bundle deploy -qq @@ -11,6 +12,7 @@ } === put and get secret in first scope + >>> [CLI] secrets put-secret test-scope-[UNIQUE_NAME]-1 my-key --string-value my-secret-value >>> [CLI] secrets get-secret test-scope-[UNIQUE_NAME]-1 my-key @@ -52,6 +54,7 @@ } === update the name of the scope (should recreate) + >>> [CLI] bundle plan -o json >>> [CLI] bundle deploy -qq @@ -65,6 +68,7 @@ } === put and get secret in recreated scope + >>> [CLI] secrets put-secret test-scope-[UNIQUE_NAME]-2 another-key --string-value another-secret-value >>> [CLI] secrets get-secret test-scope-[UNIQUE_NAME]-2 another-key @@ -89,6 +93,7 @@ } === verify there's no persistent drift + >>> [CLI] bundle plan -o json >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/output.txt b/acceptance/bundle/resources/secret_scopes/delete_scope/output.txt index 8540e39661a..fa310643c28 100644 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/output.txt +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/output.txt @@ -1,5 +1,6 @@ === create the secret scope + >>> [CLI] bundle deploy -qq >>> [CLI] bundle deploy -qq diff --git a/acceptance/bundle/resources/secret_scopes/deleted_remotely/output.txt b/acceptance/bundle/resources/secret_scopes/deleted_remotely/output.txt index 07ff6eaf942..7e8618fbb9b 100644 --- a/acceptance/bundle/resources/secret_scopes/deleted_remotely/output.txt +++ b/acceptance/bundle/resources/secret_scopes/deleted_remotely/output.txt @@ -2,9 +2,11 @@ >>> [CLI] bundle deploy -qq === delete the scope outside of the bundle + >>> [CLI] secrets delete-scope test-scope === plan should recreate the scope + >>> [CLI] bundle plan create secret_scopes.my_scope create secret_scopes.my_scope.permissions @@ -12,6 +14,7 @@ create secret_scopes.my_scope.permissions Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged === deploy should recreate the scope + >>> [CLI] bundle deploy -qq >>> [CLI] secrets list-scopes -o json diff --git a/acceptance/bundle/resources/secret_scopes/permissions-collapse/output.txt b/acceptance/bundle/resources/secret_scopes/permissions-collapse/output.txt index a26b36f3e3d..e59eb00ada5 100644 --- a/acceptance/bundle/resources/secret_scopes/permissions-collapse/output.txt +++ b/acceptance/bundle/resources/secret_scopes/permissions-collapse/output.txt @@ -1,5 +1,6 @@ === create secret scope with duplicate permissions + >>> [CLI] bundle plan >>> [CLI] bundle deploy @@ -16,6 +17,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === verify permissions are collapsed correctly + >>> [CLI] secrets list-acls test-scope-collapse-[UNIQUE_NAME] {"permission":"MANAGE","principal":"[USERNAME]"} {"permission":"WRITE","principal":"deco-test-user@databricks.com"} diff --git a/acceptance/bundle/resources/secret_scopes/permissions/output.txt b/acceptance/bundle/resources/secret_scopes/permissions/output.txt index 998becfd22f..1c8da7bd901 100644 --- a/acceptance/bundle/resources/secret_scopes/permissions/output.txt +++ b/acceptance/bundle/resources/secret_scopes/permissions/output.txt @@ -3,6 +3,7 @@ "scopes-permissions-group-[UNIQUE_NAME]" === create secret scope with permissions + >>> [CLI] bundle plan >>> [CLI] bundle deploy -qq @@ -20,6 +21,7 @@ {"permission":"WRITE","principal":"scopes-permissions-group-[UNIQUE_NAME]"} === update permissions (remove sp, change permission for user, add users and remove deco-test-group) + >>> [CLI] bundle plan >>> [CLI] bundle deploy -qq @@ -30,6 +32,7 @@ {"permission":"WRITE","principal":"deco-test-user@databricks.com"} === change scope name (should recreate scope and update permissions) + >>> [CLI] bundle plan >>> [CLI] bundle deploy -qq diff --git a/acceptance/bundle/resources/secrets/basic/output.txt b/acceptance/bundle/resources/secrets/basic/output.txt index f501c7c28e2..6a07b6135f9 100644 --- a/acceptance/bundle/resources/secrets/basic/output.txt +++ b/acceptance/bundle/resources/secrets/basic/output.txt @@ -54,6 +54,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged secrets secret1 catalog_name='main' schema_name='default' name='test_secret_[UNIQUE_NAME]' comment='Test secret for acceptance testing' value='' effective_value=None === Update comment and re-deploy + >>> update_file.py databricks.yml Test secret for acceptance testing Updated comment >>> [CLI] bundle deploy --var secret_value=initial-secret-value @@ -80,6 +81,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged secrets secret1 catalog_name='main' schema_name='default' name='test_secret_[UNIQUE_NAME]' comment='Updated comment' value='' effective_value=None === Restore comment and re-deploy + >>> update_file.py databricks.yml Updated comment Test secret for acceptance testing >>> [CLI] bundle deploy --var secret_value=initial-secret-value diff --git a/acceptance/bundle/resources/secrets/remove-expire-time/output.txt b/acceptance/bundle/resources/secrets/remove-expire-time/output.txt index 7807d0f3646..3c887f36ef4 100644 --- a/acceptance/bundle/resources/secrets/remove-expire-time/output.txt +++ b/acceptance/bundle/resources/secrets/remove-expire-time/output.txt @@ -20,6 +20,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged secrets secret1 name='test_secret_[UNIQUE_NAME]' expire_time='[EXPIRE_TIME_BEFORE]' === Remove expire_time and re-deploy + >>> update_file.py databricks.yml expire_time: "[EXPIRE_TIME_BEFORE]" >>> [CLI] bundle plan --var secret_value=secret-value @@ -53,6 +54,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged secrets secret1 name='test_secret_[UNIQUE_NAME]' expire_time=None === Plan again to record any drift the removal leaves behind + >>> [CLI] bundle plan --var secret_value=secret-value update secrets.secret1 diff --git a/acceptance/bundle/resources/secrets/update-expire-time/output.txt b/acceptance/bundle/resources/secrets/update-expire-time/output.txt index b852342db4f..cca4505055a 100644 --- a/acceptance/bundle/resources/secrets/update-expire-time/output.txt +++ b/acceptance/bundle/resources/secrets/update-expire-time/output.txt @@ -20,6 +20,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged secrets secret1 name='test_secret_[UNIQUE_NAME]' expire_time='[EXPIRE_TIME_BEFORE]' === Change expire_time and re-deploy + >>> update_file.py databricks.yml [EXPIRE_TIME_BEFORE] [EXPIRE_TIME_AFTER] >>> [CLI] bundle plan --var secret_value=secret-value diff --git a/acceptance/bundle/resources/secrets/update-value/output.txt b/acceptance/bundle/resources/secrets/update-value/output.txt index 8f12f1e007a..7c8afa1d6d1 100644 --- a/acceptance/bundle/resources/secrets/update-value/output.txt +++ b/acceptance/bundle/resources/secrets/update-value/output.txt @@ -40,6 +40,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Test secret' value='' === Update secret value by passing a different variable value + >>> [CLI] bundle plan --var secret_value=updated-secret-value -o json { "plan_version": [PLAN_VERSION], @@ -155,6 +156,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Test secret' value='' === Verify state does not contain actual secret value + >>> print_state.py { "resources.secrets.secret1": { diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/output.txt b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/output.txt index a8242e1fec1..8ed8f787978 100644 --- a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/output.txt +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/output.txt @@ -1,5 +1,6 @@ === Deploy should fail when variable used for secret has a default value + >>> [CLI] bundle deploy Error: Variable used for secret value must not have a default value at resources.secrets.secret1.value diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/output.txt b/acceptance/bundle/resources/secrets/validate-no-plain-text/output.txt index 582ae14bb48..f324d89173d 100644 --- a/acceptance/bundle/resources/secrets/validate-no-plain-text/output.txt +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/output.txt @@ -1,5 +1,6 @@ === Deploy should fail due to plain text secret value + >>> [CLI] bundle deploy Error: Secret value must be a variable reference at resources.secrets.secret1.value diff --git a/acceptance/bundle/resources/secrets/validate-secret-is-string/output.txt b/acceptance/bundle/resources/secrets/validate-secret-is-string/output.txt index 07cfd2978b5..bcf66e182cc 100644 --- a/acceptance/bundle/resources/secrets/validate-secret-is-string/output.txt +++ b/acceptance/bundle/resources/secrets/validate-secret-is-string/output.txt @@ -1,5 +1,6 @@ === Deploy should fail due to plain text secret value + >>> [CLI] bundle deploy Warning: expected string, found sequence at resources.secrets.secret1.value diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/output.txt b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/output.txt index 4e16305a146..a3e341cec11 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/output.txt +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/output.txt @@ -1,5 +1,6 @@ === Deploy with started=false: warehouse created and then stopped + >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created sql_warehouses.mywarehouse @@ -29,6 +30,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "STOPPED" === Edit a field with started=false: Edit restarts the warehouse, so a Stop must follow + >>> update_file.py databricks.yml auto_stop_mins: 10 auto_stop_mins: 20 >>> errcode [CLI] bundle deploy diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/output.txt b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/output.txt index c0eaf7285cc..5cadd8c5cc9 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/output.txt +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/output.txt @@ -1,5 +1,6 @@ === Deploy with started=false: warehouse created and then stopped + >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created sql_warehouses.mywarehouse @@ -29,6 +30,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "STOPPED" === Toggle started=false -> started=true: only Start should be called, no Edit + >>> update_file.py databricks.yml started: false started: true >>> errcode [CLI] bundle deploy @@ -48,6 +50,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "RUNNING" === Toggle started=true -> started=false: only Stop should be called, no Edit + >>> update_file.py databricks.yml started: true started: false >>> errcode [CLI] bundle deploy diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/output.txt b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/output.txt index 13b09684797..995b3141c91 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/output.txt +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/output.txt @@ -1,5 +1,6 @@ === Deploy with started=true: warehouse created and running + >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Created sql_warehouses.mywarehouse @@ -24,6 +25,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "RUNNING" === Stop warehouse externally while config says started=true: plan detects drift + >>> errcode [CLI] warehouses stop [WAREHOUSE_ID] "STOPPED" @@ -33,6 +35,7 @@ update sql_warehouses.mywarehouse Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged === Deploy fixes the drift: warehouse restarted + >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Updated sql_warehouses.mywarehouse @@ -55,6 +58,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "RUNNING" === Deploy with started=false while warehouse is stopped: warehouse stays stopped + >>> errcode [CLI] warehouses stop [WAREHOUSE_ID] "STOPPED" @@ -76,6 +80,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged "STOPPED" === Deploy with started=true: warehouse restarted + >>> update_file.py databricks.yml started: false started: true >>> errcode [CLI] bundle deploy diff --git a/acceptance/bundle/resources/sql_warehouses/output.txt b/acceptance/bundle/resources/sql_warehouses/output.txt index 5a109c71211..917c41bd0fd 100644 --- a/acceptance/bundle/resources/sql_warehouses/output.txt +++ b/acceptance/bundle/resources/sql_warehouses/output.txt @@ -41,6 +41,7 @@ Files: 10 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Verify the create request + >>> jq select(.method == "POST" and (.path | contains("/sql/warehouses"))) [OUTPUT_DIR]/out.requests.txt { "method": "POST", @@ -70,6 +71,7 @@ Resources: URL: [DATABRICKS_URL]/sql/warehouses/[UUID]?w=[WORKSPACE_ID] === Update the warehouse name + >>> update_file.py databricks.yml sql_warehouse_name sql_warehouse_name_2 >>> [CLI] bundle deploy @@ -79,6 +81,7 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 0 unchanged === Verify the update request + >>> jq select(.method == "POST" and (.path | contains("/sql/warehouses")) and (.body."name" == "sql_warehouse_name_2")) | del(.body.creator_name, .body.warehouse_type) [OUTPUT_DIR]/out.requests.txt { "method": "POST", @@ -108,6 +111,7 @@ Resources: URL: [DATABRICKS_URL]/sql/warehouses/[UUID]?w=[WORKSPACE_ID] === Destroy the warehouse + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.sql_warehouses.test_sql_warehouse @@ -117,6 +121,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted === Verify the destroy request + >>> jq select(.method == "DELETE" and (.path | contains("/sql/warehouses"))) [OUTPUT_DIR]/out.requests.txt { "method": "DELETE", diff --git a/acceptance/bundle/resources/synced_database_tables/recreate/output.txt b/acceptance/bundle/resources/synced_database_tables/recreate/output.txt index 03e95138a2f..862a94d9888 100644 --- a/acceptance/bundle/resources/synced_database_tables/recreate/output.txt +++ b/acceptance/bundle/resources/synced_database_tables/recreate/output.txt @@ -8,6 +8,7 @@ Files: 2 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Change an immutable synced-table field: no update API, so the plan recreates it + >>> [CLI] bundle plan recreate synced_database_tables.my_synced_table diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/output.txt b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/output.txt index 4c2d7b3ddeb..2aff6d57a60 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/output.txt +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/output.txt @@ -1,5 +1,6 @@ === Initial deployment (no budget_policy_id) + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-endpoint-budget-[UNIQUE_NAME]/default/files... Created vector_search_endpoints.my_endpoint @@ -7,12 +8,14 @@ Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Simulate remote drift: set budget_policy_id outside the bundle + >>> [CLI] vector-search-endpoints update-endpoint-budget-policy vs-endpoint-[UNIQUE_NAME] remote-policy { "effective_budget_policy_id": "remote-policy" } === budget_policy_id drift is ignored (effective vs requested mismatch) + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/output.txt b/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/output.txt index 94f4e96054c..8a3cba8117b 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/output.txt +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-endpoint-recreated-same-name-[UNIQUE_NAME]/default/files... Created vector_search_endpoints.my_endpoint @@ -14,6 +15,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Delete and recreate remotely with the same name + >>> [CLI] vector-search-endpoints delete-endpoint vs-endpoint-[UNIQUE_NAME] >>> [CLI] vector-search-endpoints create-endpoint vs-endpoint-[UNIQUE_NAME] STANDARD @@ -33,6 +35,7 @@ Original endpoint UUID: [ORIGINAL_ENDPOINT_UUID] Remote recreated endpoint UUID: [REMOTE_RECREATED_ENDPOINT_UUID] === Plan after out-of-band recreate + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -49,6 +52,7 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged } === Verify no permanent drift after deploy + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/output.txt b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/output.txt index 8e7a03dce0d..99b1f07096e 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/output.txt +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-endpoint-target-qps-[UNIQUE_NAME]/default/files... Created vector_search_endpoints.my_endpoint @@ -7,6 +8,7 @@ Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Simulate remote drift: change target_qps to 5 outside the bundle + >>> [CLI] vector-search-endpoints patch-endpoint vs-endpoint-[UNIQUE_NAME] --target-qps 5 { "creation_timestamp": [UNIX_TIME_MILLIS][0], @@ -25,12 +27,14 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Plan detects drift and proposes update + >>> [CLI] bundle plan update vector_search_endpoints.my_endpoint Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged === Deploy restores target_qps to 1 + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-endpoint-target-qps-[UNIQUE_NAME]/default/files... Updated vector_search_endpoints.my_endpoint diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/output.txt b/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/output.txt index c2ae0b7a92a..906b8045fb6 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/output.txt +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/output.txt @@ -1,5 +1,6 @@ === Initial deploy creates two endpoints with distinct names + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-create-fails-[UNIQUE_NAME]/default/files... Created vector_search_endpoints.blocker_endpoint @@ -8,11 +9,13 @@ Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Edit my_endpoint: rename onto blocker_endpoint's name and switch endpoint_type to trigger Recreate + >>> update_file.py databricks.yml vs-endpoint-a-[UNIQUE_NAME] vs-endpoint-b-[UNIQUE_NAME] >>> update_file.py databricks.yml endpoint_type: STANDARD endpoint_type: STORAGE_OPTIMIZED === Deploy: Recreate of my_endpoint runs Delete (ok) then Create (409, name taken by blocker) + >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-create-fails-[UNIQUE_NAME]/default/files... Error: cannot recreate resources.vector_search_endpoints.my_endpoint: Vector search endpoint with name vs-endpoint-b-[UNIQUE_NAME] already exists (409 RESOURCE_ALREADY_EXISTS) @@ -27,6 +30,7 @@ Files: 0 uploaded, 0 deleted Exit code: 1 === Subsequent plan recovers: my_endpoint state was dropped, replan as Create + >>> [CLI] bundle plan create vector_search_endpoints.my_endpoint diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/output.txt b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/output.txt index 22ca8a55cf6..cd0110b6f08 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/output.txt +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/output.txt @@ -1,5 +1,6 @@ === Initial deployment with STANDARD endpoint_type + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-vs-endpoint-[UNIQUE_NAME]/default/files... Created vector_search_endpoints.my_endpoint @@ -9,6 +10,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --keep //vector-search/endpoints === Change endpoint_type (should trigger recreation) + >>> update_file.py databricks.yml endpoint_type: STANDARD endpoint_type: STORAGE_OPTIMIZED >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/output.txt b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/output.txt index 017c9052b9b..893cd303576 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/output.txt +++ b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/output.txt @@ -1,5 +1,6 @@ === Initial deployment (no budget policy) + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-vs-endpoint-budget-[UNIQUE_NAME]/default/files... Created vector_search_endpoints.my_endpoint @@ -9,6 +10,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --keep //vector-search/endpoints === Add budget_policy_id + >>> update_file.py databricks.yml endpoint_type: STANDARD endpoint_type: STANDARD budget_policy_id: test-policy-id diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/output.txt b/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/output.txt index 44bbd77a910..a94b54be906 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/output.txt +++ b/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-vs-endpoint-target-qps-[UNIQUE_NAME]/default/files... Created vector_search_endpoints.my_endpoint @@ -16,6 +17,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Update target_qps + >>> update_file.py databricks.yml target_qps: 1 target_qps: 2 >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt index b5aed53ad30..1dc6402e1bc 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-deleted-[UNIQUE_NAME]/default/files... Created vector_search_endpoints.my_endpoint @@ -8,15 +9,18 @@ Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Simulate remote deletion + >>> [CLI] vector-search-indexes delete-index main.default.vs_index_[UNIQUE_NAME] === Plan detects missing resource and proposes creation + >>> [CLI] bundle plan create vector_search_indexes.my_index Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged === Deploy recreates the index + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-deleted-[UNIQUE_NAME]/default/files... Created vector_search_indexes.my_index diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt index 2f0118a9743..b3b5050c5ba 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-orphaned-endpoint-[UNIQUE_NAME]/default/files... Created vector_search_endpoints.vs_endpoint @@ -8,9 +9,11 @@ Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Simulate remote endpoint deletion (index is orphaned, still exists by name) + >>> [CLI] vector-search-endpoints delete-endpoint vs-endpoint-[UNIQUE_NAME] === Plan must cascade-recreate the index because its endpoint is being recreated + >>> [CLI] bundle plan create vector_search_endpoints.vs_endpoint recreate vector_search_indexes.my_index @@ -18,6 +21,7 @@ recreate vector_search_indexes.my_index Plan: 2 to add, 0 to change, 1 to delete, 0 unchanged === Deploy recreates endpoint and rebinds index + >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-index-orphaned-endpoint-[UNIQUE_NAME]/default/files... diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/output.txt index cdb909152fd..d009af5482b 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files... Created vector_search_endpoints.my_endpoint @@ -10,6 +11,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --get --unique //vector-search/indexes === Change embedding_dimension (should trigger recreation) + >>> update_file.py databricks.yml embedding_dimension: 768 embedding_dimension: 384 >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/output.txt index a175c2dcae3..50e9947cd11 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/output.txt @@ -1,5 +1,6 @@ === Initial deployment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-[UNIQUE_NAME]/default/files... Created vector_search_endpoints.my_endpoint @@ -8,6 +9,7 @@ Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Change embedding_dimension (recreate: DELETE then CREATE the same name) + >>> update_file.py databricks.yml embedding_dimension: 768 embedding_dimension: 384 >>> [CLI] bundle deploy --auto-approve diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt index f704efd804d..8459392750f 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt @@ -1,5 +1,6 @@ === Initial deployment with STANDARD endpoint_type + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-index-with-endpoint-[UNIQUE_NAME]/default/files... Created vector_search_endpoints.my_endpoint @@ -8,6 +9,7 @@ Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Change endpoint_type (recreates endpoint; index recreate cascade is not yet implemented) + >>> update_file.py databricks.yml endpoint_type: STANDARD endpoint_type: STORAGE_OPTIMIZED >>> [CLI] bundle plan @@ -128,6 +130,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } === Re-plan exposes the leftover drift: the index still points at the old endpoint UUID + >>> [CLI] bundle plan recreate vector_search_indexes.my_index diff --git a/acceptance/bundle/resources/volumes/change-comment/output.txt b/acceptance/bundle/resources/volumes/change-comment/output.txt index b045eb60df2..33fc5957861 100644 --- a/acceptance/bundle/resources/volumes/change-comment/output.txt +++ b/acceptance/bundle/resources/volumes/change-comment/output.txt @@ -1,5 +1,6 @@ === Initial summary for resource, before the deployment + >>> [CLI] bundle summary -o json { "catalog_name": "main", @@ -12,6 +13,7 @@ } === Verify volume does not exist + >>> musterr [CLI] volumes read main.myschema.myvolume Error: Volume 'main.myschema.myvolume' does not exist. @@ -39,6 +41,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "[DATABRICKS_URL]/explore/data/volumes/main/myschema/myvolume?w=[WORKSPACE_ID]" === Verify deployment + >>> [CLI] volumes read main.myschema.myvolume { "catalog_name": "main", @@ -56,6 +59,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Update comment + >>> update_file.py databricks.yml COMMENT1 COMMENT2 === Summary should show modified_status=modified and show the same id @@ -86,6 +90,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } === Verify updated deployment: should show new comment + >>> [CLI] volumes read main.myschema.myvolume "COMMENT2" diff --git a/acceptance/bundle/resources/volumes/change-comment/script b/acceptance/bundle/resources/volumes/change-comment/script index 913d5acdf90..d623c47fe12 100644 --- a/acceptance/bundle/resources/volumes/change-comment/script +++ b/acceptance/bundle/resources/volumes/change-comment/script @@ -8,7 +8,7 @@ trace $CLI bundle deploy trace print_requests.py //unity -title "Summary should now show id and url\n" +title "Summary should now show id and url" $CLI bundle summary -o json | jq .resources.volumes.volume1.id $CLI bundle summary -o json | jq .resources.volumes.volume1.url @@ -18,7 +18,7 @@ trace $CLI volumes read main.myschema.myvolume title "Update comment" trace update_file.py databricks.yml COMMENT1 COMMENT2 -title "Summary should show modified_status=modified and show the same id\n" +title "Summary should show modified_status=modified and show the same id" $CLI bundle summary -o json | jq .resources.volumes.volume1 trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/volumes/change-name/output.txt b/acceptance/bundle/resources/volumes/change-name/output.txt index 81b57fcda65..337de220ab9 100644 --- a/acceptance/bundle/resources/volumes/change-name/output.txt +++ b/acceptance/bundle/resources/volumes/change-name/output.txt @@ -35,6 +35,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Update name + >>> update_file.py databricks.yml myvolume mynewvolume >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/volumes/change-name/script b/acceptance/bundle/resources/volumes/change-name/script index e2c2c5ea7ba..d8f122a9820 100644 --- a/acceptance/bundle/resources/volumes/change-name/script +++ b/acceptance/bundle/resources/volumes/change-name/script @@ -2,7 +2,7 @@ trace $CLI bundle deploy trace print_requests.py //unity -title "Summary should now show id and url\n" +title "Summary should now show id and url" $CLI bundle summary -o json | jq .resources title "Update name" diff --git a/acceptance/bundle/resources/volumes/change-schema-name/output.txt b/acceptance/bundle/resources/volumes/change-schema-name/output.txt index 7791b740c98..f739e7ee5af 100644 --- a/acceptance/bundle/resources/volumes/change-schema-name/output.txt +++ b/acceptance/bundle/resources/volumes/change-schema-name/output.txt @@ -35,6 +35,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Update name + >>> update_file.py databricks.yml myschema mynewschema >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/volumes/change-schema-name/script b/acceptance/bundle/resources/volumes/change-schema-name/script index c369e53fff9..7de9ae73bed 100644 --- a/acceptance/bundle/resources/volumes/change-schema-name/script +++ b/acceptance/bundle/resources/volumes/change-schema-name/script @@ -2,7 +2,7 @@ trace $CLI bundle deploy trace print_requests.py //unity -title "Summary should now show id and url\n" +title "Summary should now show id and url" $CLI bundle summary -o json | jq .resources title "Update name" diff --git a/acceptance/bundle/resources/volumes/comment_out_of_band/output.txt b/acceptance/bundle/resources/volumes/comment_out_of_band/output.txt index 82af14b00a2..651e7be09bd 100644 --- a/acceptance/bundle/resources/volumes/comment_out_of_band/output.txt +++ b/acceptance/bundle/resources/volumes/comment_out_of_band/output.txt @@ -1,5 +1,6 @@ === Deploy a volume that does not declare a comment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -8,7 +9,9 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Set the comment out of band, the way Catalog Explorer does + === The remote comment is drift, so the plan updates the volume + >>> [CLI] bundle plan --output json { "comment": { @@ -23,6 +26,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Redeploy clears the comment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated volumes.volume1 @@ -30,16 +34,19 @@ Files: 0 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged === The comment is gone + >>> [CLI] api get /api/2.1/unity-catalog/volumes/main.test-schema-[UNIQUE_NAME].test-volume-[UNIQUE_NAME] "" === Redeploy is a no-op + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 2 unchanged === The UpdateVolume payloads the deploys sent + >>> print_requests.py --method PATCH //unity-catalog/volumes { "method": "PATCH", diff --git a/acceptance/bundle/resources/volumes/comment_with_rename/output.txt b/acceptance/bundle/resources/volumes/comment_with_rename/output.txt index 224755056d0..f3fba0fee56 100644 --- a/acceptance/bundle/resources/volumes/comment_with_rename/output.txt +++ b/acceptance/bundle/resources/volumes/comment_with_rename/output.txt @@ -1,5 +1,6 @@ === Deploy a volume that does not declare a comment + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created schemas.schema1 @@ -8,6 +9,7 @@ Files: 2 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Set the comment out of band, then rename the volume in the same deploy + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated volumes.volume1 @@ -15,6 +17,7 @@ Files: 1 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged === The rename and the clear both landed, so it converges in one pass + >>> [CLI] api get /api/2.1/unity-catalog/volumes/main.test-schema-[UNIQUE_NAME].vol_two "" @@ -22,6 +25,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === The rename carried the cleared comment alongside new_name + >>> print_requests.py --method PATCH //unity-catalog/volumes { "method": "PATCH", diff --git a/acceptance/bundle/resources/volumes/recreate/output.txt b/acceptance/bundle/resources/volumes/recreate/output.txt index fed59b1a868..02945297ade 100644 --- a/acceptance/bundle/resources/volumes/recreate/output.txt +++ b/acceptance/bundle/resources/volumes/recreate/output.txt @@ -11,6 +11,7 @@ Validation OK! >>> [CLI] bundle deploy -qq === Assert the volume is created successfully + >>> [CLI] volumes read main.schema1-[UNIQUE_NAME].my_volume { "catalog_name": "main", @@ -20,6 +21,7 @@ Validation OK! } === Assert that the grants were successfully applied + >>> [CLI] grants get VOLUME main.schema1-[UNIQUE_NAME].my_volume { "privilege_assignments": [ @@ -33,6 +35,7 @@ Validation OK! } === Recreation of the volume without --auto-approve should fail since prompting is not possible + >>> errcode [CLI] bundle deploy --var=schema_name=${resources.schemas.schema2.name} Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-uc-volumes-[UNIQUE_NAME]/default/files... @@ -50,6 +53,7 @@ Files: 0 uploaded, 0 deleted Exit code: 1 === Successfully recreate the volume with --auto-approve + >>> [CLI] bundle deploy --var=schema_name=${resources.schemas.schema2.name} --auto-approve -qq This action will result in the deletion or recreation of the following volumes. diff --git a/acceptance/bundle/run/app-with-job/output.txt b/acceptance/bundle/run/app-with-job/output.txt index e57ad4ae313..6df090b5b50 100644 --- a/acceptance/bundle/run/app-with-job/output.txt +++ b/acceptance/bundle/run/app-with-job/output.txt @@ -16,6 +16,7 @@ Files: 5 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === App should exist after bundle deployment + >>> [CLI] apps get app-[UNIQUE_NAME] { "app_status": { @@ -25,6 +26,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === Check app config + >>> [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/app-with-job-[UNIQUE_NAME]/files/app/app.yml command: - flask @@ -40,6 +42,7 @@ env: >>> cat LOG.run.txt === App should be in the running state + >>> [CLI] apps get app-[UNIQUE_NAME] { "app_status": { @@ -49,6 +52,7 @@ env: } === Stop the app + >>> [CLI] apps stop app-[UNIQUE_NAME] { "compute_status": { @@ -74,6 +78,7 @@ env: >>> cat LOG.run.txt === App should be in the running state + >>> [CLI] apps get app-[UNIQUE_NAME] { "app_status": { @@ -83,6 +88,7 @@ env: } === Redeploy it again just to check that it can be redeployed + >>> [CLI] bundle deploy --force-lock --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/app-with-job-[UNIQUE_NAME]/files... Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/run/app-with-job/script b/acceptance/bundle/run/app-with-job/script index f0b41a2881e..c15ba27e7ab 100644 --- a/acceptance/bundle/run/app-with-job/script +++ b/acceptance/bundle/run/app-with-job/script @@ -15,7 +15,7 @@ APP_URL=$($CLI apps get "app-${UNIQUE_NAME}" | jq -r '.url') title "Check app config" trace $CLI workspace export "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/app-with-job-${UNIQUE_NAME}/files/app/app.yml" -title "Try to run the app\n" +title "Try to run the app" $CLI bundle run test_app &> LOG.run.txt trace cat LOG.run.txt | grep -q "${APP_URL}" @@ -26,7 +26,7 @@ title "Stop the app" trace $CLI apps stop "app-${UNIQUE_NAME}" | jq '{compute_status}' trace $CLI apps get "app-${UNIQUE_NAME}" | jq '{app_status,compute_status}' -title "Try to run the app again\n" +title "Try to run the app again" $CLI bundle run test_app &> LOG.run.txt trace cat LOG.run.txt | grep -q "${APP_URL}" diff --git a/acceptance/bundle/run/basic/output.txt b/acceptance/bundle/run/basic/output.txt index 9ae6068709a..d3ab49b8603 100644 --- a/acceptance/bundle/run/basic/output.txt +++ b/acceptance/bundle/run/basic/output.txt @@ -1,11 +1,13 @@ === no run key specified + >>> [CLI] bundle run Error: expected a KEY of the resource to run Exit code: 1 === deploy and run resource + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/caterpillar/default/files... Created jobs.foo @@ -20,12 +22,14 @@ Run URL: [DATABRICKS_URL]/jobs/[FOO_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] 1 === no resource key with -- + >>> [CLI] bundle run -- Error: expected a KEY of the resource to run Exit code: 1 === resource key with parameters + >>> [CLI] bundle run foo -- arg1 arg2 Run URL: [DATABRICKS_URL]/jobs/[FOO_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] @@ -34,10 +38,12 @@ Run URL: [DATABRICKS_URL]/jobs/[FOO_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] 1 === inline script + >>> [CLI] bundle run -- echo hello hello === completion handler + >>> [CLI] __complete bundle run , foo :4 diff --git a/acceptance/bundle/run/development-mode-changes/output.txt b/acceptance/bundle/run/development-mode-changes/output.txt index fbc0856911d..9431a93f3d3 100644 --- a/acceptance/bundle/run/development-mode-changes/output.txt +++ b/acceptance/bundle/run/development-mode-changes/output.txt @@ -1,5 +1,6 @@ === mode: development with pipeline development: false: deploy and run both use development: true + >>> [CLI] bundle deploy -t dev_false Warning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes. at resources.pipelines.my_pipeline.development @@ -35,6 +36,7 @@ Update ID: [UUID][1] } === Switched to mode: development without redeploying: run sends development: true + >>> [CLI] bundle deploy -t switch_to_dev Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-dev-mode-changes/switch_to_dev/files... Created pipelines.my_pipeline @@ -58,6 +60,7 @@ Update ID: [UUID][3] } === Switched away from mode: development without redeploying: run does not send development + >>> [CLI] bundle deploy -t switch_from_dev Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-dev-mode-changes/switch_from_dev/files... Created pipelines.my_pipeline @@ -79,6 +82,7 @@ Update ID: [UUID][5] } === Pipeline switched to production outside the bundle: run on a development target sends development: true + >>> [CLI] bundle deploy -t ui_toggle Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-dev-mode-changes/ui_toggle/files... Created pipelines.my_pipeline diff --git a/acceptance/bundle/run/development-mode/output.txt b/acceptance/bundle/run/development-mode/output.txt index 8abe1170d5c..6bd0e99452d 100644 --- a/acceptance/bundle/run/development-mode/output.txt +++ b/acceptance/bundle/run/development-mode/output.txt @@ -1,5 +1,6 @@ === Production target: update does not send development + >>> [CLI] bundle deploy -t prod Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-dev-mode/prod/files... Created pipelines.my_pipeline @@ -19,6 +20,7 @@ Update ID: [UUID][1] } === Development target: update sends development: true + >>> [CLI] bundle deploy -t dev Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-dev-mode/dev/files... Created pipelines.my_pipeline @@ -40,6 +42,7 @@ Update ID: [UUID][3] } === Development target via pipelines run: update sends development: true + >>> [CLI] pipelines run my_pipeline -t dev Recommendation: This command runs the last deployed version of the code @@ -64,6 +67,7 @@ Pipeline configurations for this update: } === Development target with pipelines_development preset disabled: update does not send development + >>> [CLI] bundle deploy -t dev_preset_off Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-dev-mode/dev_preset_off/files... Created pipelines.my_pipeline diff --git a/acceptance/bundle/run/development-override/output.txt b/acceptance/bundle/run/development-override/output.txt index 85ed152bb16..125df1ba057 100644 --- a/acceptance/bundle/run/development-override/output.txt +++ b/acceptance/bundle/run/development-override/output.txt @@ -1,5 +1,6 @@ === Deploy warns about the deprecated property and passes development: true through + >>> [CLI] bundle deploy Warning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes. at resources.pipelines.my_pipeline.development @@ -17,6 +18,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Run on a non-development target does not send development; the pipeline-level value applies + >>> [CLI] bundle run my_pipeline Warning: The "development" property of pipelines is deprecated. Use "mode: development" in your databricks.yml instead. See https://docs.databricks.com/dev-tools/bundles/deployment-modes. at resources.pipelines.my_pipeline.development diff --git a/acceptance/bundle/run/job-ref-job-id/output.txt b/acceptance/bundle/run/job-ref-job-id/output.txt index b1c11ca57a2..f09ab66f3e2 100644 --- a/acceptance/bundle/run/job-ref-job-id/output.txt +++ b/acceptance/bundle/run/job-ref-job-id/output.txt @@ -7,12 +7,14 @@ Files: 1 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === foo's task points at bar's job id + >>> [CLI] jobs get [FOO_ID] { "job_id": [BAR_ID] } === run resolves the referenced job id into the int job_id field + >>> [CLI] bundle run foo Run URL: [DATABRICKS_URL]/jobs/[FOO_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] diff --git a/acceptance/bundle/run/refresh-flags/output.txt b/acceptance/bundle/run/refresh-flags/output.txt index 78680ea662f..f5b077ea23d 100644 --- a/acceptance/bundle/run/refresh-flags/output.txt +++ b/acceptance/bundle/run/refresh-flags/output.txt @@ -6,6 +6,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Running pipeline with --refresh flag and specific tables + >>> [CLI] bundle run my_pipeline --refresh table1,table2 Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][1] @@ -24,6 +25,7 @@ Update ID: [UUID][1] } === Running pipeline with --full-refresh-all flag + >>> [CLI] bundle run my_pipeline --full-refresh-all Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][2] @@ -39,6 +41,7 @@ Update ID: [UUID][2] } === Running pipeline with --full-refresh flag and specific tables + >>> [CLI] bundle run my_pipeline --full-refresh table1,table2 Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][3] @@ -57,6 +60,7 @@ Update ID: [UUID][3] } === Running pipeline with --full-refresh flag and --refresh flag + >>> [CLI] bundle run my_pipeline --full-refresh table1,table2 --refresh table3,table4 Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][4] diff --git a/acceptance/bundle/run/state-wiped/output.txt b/acceptance/bundle/run/state-wiped/output.txt index 894e4148455..9ce9243f804 100644 --- a/acceptance/bundle/run/state-wiped/output.txt +++ b/acceptance/bundle/run/state-wiped/output.txt @@ -16,6 +16,7 @@ Run URL: [DATABRICKS_URL]/jobs/[FOO_ID]/runs/[RUN_ID]?o=[WORKSPACE_ID] [TIMESTAMP] "foo" TERMINATED SUCCESS === Testing that clean state that affect run command -- it'll fetch the state + >>> rm -fr .databricks >>> [CLI] bundle run foo diff --git a/acceptance/bundle/run_as/allowed/regular_user/output.txt b/acceptance/bundle/run_as/allowed/regular_user/output.txt index a326e670813..03d36c3c76f 100644 --- a/acceptance/bundle/run_as/allowed/regular_user/output.txt +++ b/acceptance/bundle/run_as/allowed/regular_user/output.txt @@ -7,6 +7,7 @@ } === Jobs one and two run_as is updated to bundle-level setting (service_principal_name) + >>> jq .resources.jobs.job_one.run_as { "service_principal_name": "my_service_principal" @@ -18,6 +19,7 @@ } === Other resources are not affected + >>> jq .resources.jobs.job_three.run_as { "service_principal_name": "my_service_principal_for_job" @@ -45,6 +47,7 @@ } === Jobs one and two run_as is updated to target-level setting (user_name) + >>> jq .resources.jobs.job_one.run_as { "user_name": "my_user_name" @@ -56,6 +59,7 @@ } === Other resources are not affected + >>> jq .resources.jobs.job_three.run_as { "service_principal_name": "my_service_principal_for_job" diff --git a/acceptance/bundle/run_as/allowed/service_principal/output.txt b/acceptance/bundle/run_as/allowed/service_principal/output.txt index a326e670813..03d36c3c76f 100644 --- a/acceptance/bundle/run_as/allowed/service_principal/output.txt +++ b/acceptance/bundle/run_as/allowed/service_principal/output.txt @@ -7,6 +7,7 @@ } === Jobs one and two run_as is updated to bundle-level setting (service_principal_name) + >>> jq .resources.jobs.job_one.run_as { "service_principal_name": "my_service_principal" @@ -18,6 +19,7 @@ } === Other resources are not affected + >>> jq .resources.jobs.job_three.run_as { "service_principal_name": "my_service_principal_for_job" @@ -45,6 +47,7 @@ } === Jobs one and two run_as is updated to target-level setting (user_name) + >>> jq .resources.jobs.job_one.run_as { "user_name": "my_user_name" @@ -56,6 +59,7 @@ } === Other resources are not affected + >>> jq .resources.jobs.job_three.run_as { "service_principal_name": "my_service_principal_for_job" diff --git a/acceptance/bundle/run_as/job_default/output.txt b/acceptance/bundle/run_as/job_default/output.txt index e8fa5630934..fffe74df026 100644 --- a/acceptance/bundle/run_as/job_default/output.txt +++ b/acceptance/bundle/run_as/job_default/output.txt @@ -1,5 +1,6 @@ === Deploy with run_as + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/run_as_job_default_[UNIQUE_NAME]/default/files... Created jobs.job_with_run_as @@ -47,6 +48,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === Remove run_as and redeploy + >>> [CLI] bundle plan update jobs.job_with_run_as diff --git a/acceptance/bundle/run_as/pipelines/regular_user/output.txt b/acceptance/bundle/run_as/pipelines/regular_user/output.txt index 21faafb4d57..20718c7be42 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/output.txt +++ b/acceptance/bundle/run_as/pipelines/regular_user/output.txt @@ -1,5 +1,6 @@ -=== run_as should succeed in all cases\nProcessing target: t_user_name +=== run_as should succeed in all cases +\nProcessing target: t_user_name >>> [CLI] bundle validate -t t_user_name Name: run_as diff --git a/acceptance/bundle/run_as/pipelines/service_principal/output.txt b/acceptance/bundle/run_as/pipelines/service_principal/output.txt index 21faafb4d57..20718c7be42 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/output.txt +++ b/acceptance/bundle/run_as/pipelines/service_principal/output.txt @@ -1,5 +1,6 @@ -=== run_as should succeed in all cases\nProcessing target: t_user_name +=== run_as should succeed in all cases +\nProcessing target: t_user_name >>> [CLI] bundle validate -t t_user_name Name: run_as diff --git a/acceptance/bundle/run_as/pipelines_legacy/output.txt b/acceptance/bundle/run_as/pipelines_legacy/output.txt index cfd58c5e866..b8500062649 100644 --- a/acceptance/bundle/run_as/pipelines_legacy/output.txt +++ b/acceptance/bundle/run_as/pipelines_legacy/output.txt @@ -11,6 +11,7 @@ Warning: You are using the legacy mode of run_as. The support for this mode is e } === Jobs one and two run_as is updated to bundle-level setting (service_principal_name) + >>> jq .resources.jobs.job_one.run_as { "service_principal_name": "my_service_principal" @@ -22,6 +23,7 @@ Warning: You are using the legacy mode of run_as. The support for this mode is e } === Pipeline's CAN_VIEW permissions for my_service_principal is upgraded to IS_OWNER + >>> jq .resources.pipelines.nyc_taxi_pipeline.permissions [ { @@ -39,6 +41,7 @@ Warning: You are using the legacy mode of run_as. The support for this mode is e ] === Other resources are not affected + >>> jq .resources.jobs.job_three.run_as { "service_principal_name": "my_service_principal_for_job" diff --git a/acceptance/bundle/scripts/restricted-execution/output.txt b/acceptance/bundle/scripts/restricted-execution/output.txt index 57cbd1e88e3..1fe2aa810a6 100644 --- a/acceptance/bundle/scripts/restricted-execution/output.txt +++ b/acceptance/bundle/scripts/restricted-execution/output.txt @@ -1,5 +1,6 @@ === Without DATABRICKS_BUNDLE_RESTRICTED_CODE_EXECUTION, all envs are accessible + >>> errcode [CLI] bundle deploy Executing 'preinit' script preinit value_from_env @@ -18,6 +19,7 @@ Executing 'postdeploy' script postdeploy value_from_env === With DATABRICKS_BUNDLE_RESTRICTED_CODE_EXECUTION=1, no envs are accessible + >>> DATABRICKS_BUNDLE_RESTRICTED_CODE_EXECUTION=1 errcode [CLI] bundle deploy Error: failed to execute script: running scripts is not allowed when DATABRICKS_BUNDLE_RESTRICTED_CODE_EXECUTION is set diff --git a/acceptance/bundle/select/basic/output.txt b/acceptance/bundle/select/basic/output.txt index 89f66b3d78f..a50e47392f2 100644 --- a/acceptance/bundle/select/basic/output.txt +++ b/acceptance/bundle/select/basic/output.txt @@ -95,6 +95,7 @@ Resources: URL: [DATABRICKS_URL]/jobs/[FOO_ID]?w=[WORKSPACE_ID] === Full plan after partial deploy + >>> [CLI] bundle plan create jobs.baz @@ -126,10 +127,12 @@ Resources: 1 created, 0 changed, 0 deleted, 2 unchanged } === Full plan again + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged === Destroy + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.bar diff --git a/acceptance/bundle/select/basic/script b/acceptance/bundle/select/basic/script index 3b7aa10b34d..46ac51e0459 100644 --- a/acceptance/bundle/select/basic/script +++ b/acceptance/bundle/select/basic/script @@ -23,11 +23,11 @@ trace $CLI bundle plan --select jobs.foo # inline, or via --plan (READPLAN=1). The deploy is not traced because readplanarg # varies the command line between READPLAN variants, which must produce identical output. $CLI bundle plan --select jobs.foo -o json > plan.json -title "bundle deploy --select jobs.foo\n" +title "bundle deploy --select jobs.foo" $CLI bundle deploy --select jobs.foo $(readplanarg plan.json) replace_ids.py # The deploy reports that --select was used via telemetry. -title "Telemetry:\n" +title "Telemetry:" print_telemetry_bool_values | grep '^select_used ' # Only bar and foo were created, never baz. trace print_requests.py --nostamp --sort //jobs @@ -39,7 +39,7 @@ trace $CLI bundle summary title "Full plan after partial deploy" trace $CLI bundle plan $CLI bundle plan -o json > plan-full.json -title "Full deploy\n" +title "Full deploy" $CLI bundle deploy $(readplanarg plan-full.json) # Only baz is created this time. trace print_requests.py --nostamp --sort //jobs diff --git a/acceptance/bundle/select/deletion/output.txt b/acceptance/bundle/select/deletion/output.txt index a1083051d83..dd55a908b92 100644 --- a/acceptance/bundle/select/deletion/output.txt +++ b/acceptance/bundle/select/deletion/output.txt @@ -67,6 +67,7 @@ Resources: 0 created, 0 changed, 1 deleted, 0 unchanged, 1 not selected Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Destroy + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.keep diff --git a/acceptance/bundle/select/deletion/script b/acceptance/bundle/select/deletion/script index c495b8dac39..c7c90841ce1 100644 --- a/acceptance/bundle/select/deletion/script +++ b/acceptance/bundle/select/deletion/script @@ -8,7 +8,7 @@ cleanup() { trap cleanup EXIT # Deploy both jobs. -title "Initial deploy\n" +title "Initial deploy" trace $CLI bundle deploy # Register [KEEP_ID]/[GONE_ID] from state while both still exist, so the delete # request body below is masked deterministically. @@ -21,7 +21,7 @@ envsubst '$UNIQUE_NAME' < databricks.yml.after.tmpl > databricks.yml # one. The plan is filtered to just its deletion. trace $CLI bundle plan --select jobs.gone -title "bundle deploy --select jobs.gone\n" +title "bundle deploy --select jobs.gone" trace $CLI bundle deploy --select jobs.gone # The delete call targeted 'gone' only. @@ -32,7 +32,7 @@ trace $CLI bundle plan # The same works with an unqualified name. Deploy 'gone' again, then remove it from # config so it is once more orphaned in state. -title "Redeploy gone, then select it by unqualified name\n" +title "Redeploy gone, then select it by unqualified name" envsubst '$UNIQUE_NAME' < databricks.yml.tmpl > databricks.yml trace $CLI bundle deploy replace_ids.py @@ -40,7 +40,7 @@ envsubst '$UNIQUE_NAME' < databricks.yml.after.tmpl > databricks.yml trace $CLI bundle plan --select gone -title "bundle deploy --select gone\n" +title "bundle deploy --select gone" trace $CLI bundle deploy --select gone trace print_requests.py --sort //jobs/delete diff --git a/acceptance/bundle/select/deletion_dependents/output.txt b/acceptance/bundle/select/deletion_dependents/output.txt index c58c0895137..f3b06692b36 100644 --- a/acceptance/bundle/select/deletion_dependents/output.txt +++ b/acceptance/bundle/select/deletion_dependents/output.txt @@ -62,6 +62,7 @@ Resources: 0 created, 0 changed, 2 deleted, 0 unchanged, 1 not selected Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Destroy + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.keep diff --git a/acceptance/bundle/select/deletion_dependents/script b/acceptance/bundle/select/deletion_dependents/script index d2c1dab89f5..b8e3b571628 100644 --- a/acceptance/bundle/select/deletion_dependents/script +++ b/acceptance/bundle/select/deletion_dependents/script @@ -9,7 +9,7 @@ trap cleanup EXIT # Deploy three jobs; consumer references producer, so state records consumer's # dependency on producer. -title "Initial deploy\n" +title "Initial deploy" trace $CLI bundle deploy replace_ids.py @@ -38,7 +38,7 @@ envsubst '$UNIQUE_NAME' < databricks.yml.after.tmpl > databricks.yml # untouched. The dependency direction is reversed for deletes. trace $CLI bundle plan --select jobs.producer -title "bundle deploy --select jobs.producer\n" +title "bundle deploy --select jobs.producer" trace $CLI bundle deploy --select jobs.producer # Both producer and consumer were deleted, dependent first; keep was not. Not sorted: diff --git a/acceptance/bundle/select/deletion_saved_plan/output.txt b/acceptance/bundle/select/deletion_saved_plan/output.txt index 026d4672a6b..00f65e16370 100644 --- a/acceptance/bundle/select/deletion_saved_plan/output.txt +++ b/acceptance/bundle/select/deletion_saved_plan/output.txt @@ -29,6 +29,7 @@ Resources: 0 created, 0 changed, 1 deleted, 0 unchanged, 1 not selected Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Destroy + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.keep diff --git a/acceptance/bundle/select/deletion_saved_plan/script b/acceptance/bundle/select/deletion_saved_plan/script index 8575df90011..acbf37dd30a 100644 --- a/acceptance/bundle/select/deletion_saved_plan/script +++ b/acceptance/bundle/select/deletion_saved_plan/script @@ -8,7 +8,7 @@ cleanup() { trap cleanup EXIT # Deploy both jobs. -title "Initial deploy\n" +title "Initial deploy" trace $CLI bundle deploy # Register [KEEP_ID]/[GONE_ID] from state while both still exist, so the delete # request body below is masked deterministically. @@ -24,7 +24,7 @@ $CLI bundle plan --select jobs.gone -o json > plan.json # Deploy from the plan file WITHOUT re-passing --select: the deletion was selected # at plan time and survives serialization, so applying the saved plan deletes 'gone' # and reports the ", 1 not selected" suffix from plan.NotSelected, not the flag. -title "bundle deploy --plan (no --select)\n" +title "bundle deploy --plan (no --select)" trace $CLI bundle deploy --plan plan.json # The delete call targeted 'gone' only. diff --git a/acceptance/bundle/select/grants_permissions/output.txt b/acceptance/bundle/select/grants_permissions/output.txt index 6ada2c1689b..9f8b0f113ce 100644 --- a/acceptance/bundle/select/grants_permissions/output.txt +++ b/acceptance/bundle/select/grants_permissions/output.txt @@ -66,6 +66,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged, 2 not selected } === Destroy + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.my_job diff --git a/acceptance/bundle/select/grants_permissions/script b/acceptance/bundle/select/grants_permissions/script index 7856742c6cf..88195b2d5d4 100644 --- a/acceptance/bundle/select/grants_permissions/script +++ b/acceptance/bundle/select/grants_permissions/script @@ -19,11 +19,11 @@ musterr trace $CLI bundle plan --select schemas.my_schema.grants # Deploying a selected resource applies its permissions/grants too. $CLI bundle plan --select jobs.my_job -o json > plan-job.json -title "bundle deploy --select jobs.my_job\n" +title "bundle deploy --select jobs.my_job" $CLI bundle deploy --select jobs.my_job $(readplanarg plan-job.json) $CLI bundle plan --select schemas.my_schema -o json > plan-schema.json -title "bundle deploy --select schemas.my_schema\n" +title "bundle deploy --select schemas.my_schema" $CLI bundle deploy --select schemas.my_schema $(readplanarg plan-schema.json) replace_ids.py diff --git a/acceptance/bundle/select/plan_file_survives_not_selected/script b/acceptance/bundle/select/plan_file_survives_not_selected/script index 908aeb6ad1d..7a2e6443ddd 100644 --- a/acceptance/bundle/select/plan_file_survives_not_selected/script +++ b/acceptance/bundle/select/plan_file_survives_not_selected/script @@ -12,5 +12,5 @@ $CLI bundle plan --select jobs.foo -o json > plan.json # Deploy from the plan file WITHOUT re-passing --select. b.Select is empty here, # so the ", 2 not selected" suffix must come from the serialized plan.NotSelected, # not from the flag. This is the regression guard for that wiring. -title "bundle deploy --plan (no --select)\n" +title "bundle deploy --plan (no --select)" trace $CLI bundle deploy --plan plan.json diff --git a/acceptance/bundle/select/redeploy_unqualified/output.txt b/acceptance/bundle/select/redeploy_unqualified/output.txt index b09c02e30e2..a06cfaae023 100644 --- a/acceptance/bundle/select/redeploy_unqualified/output.txt +++ b/acceptance/bundle/select/redeploy_unqualified/output.txt @@ -15,6 +15,7 @@ Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged === Destroy + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.my_job diff --git a/acceptance/bundle/select/redeploy_unqualified/script b/acceptance/bundle/select/redeploy_unqualified/script index f99a7543e4d..8e75bd70fca 100644 --- a/acceptance/bundle/select/redeploy_unqualified/script +++ b/acceptance/bundle/select/redeploy_unqualified/script @@ -6,10 +6,10 @@ cleanup() { } trap cleanup EXIT -title "Initial deploy\n" +title "Initial deploy" trace $CLI bundle deploy # Redeploy selecting the resource by its unqualified name. It exists in both config # and state now; that must not read as an ambiguous match. -title "Redeploy with unqualified --select\n" +title "Redeploy with unqualified --select" trace $CLI bundle deploy --select my_job diff --git a/acceptance/bundle/state/engine_default/script b/acceptance/bundle/state/engine_default/script index cadf288994a..02a757f356c 100644 --- a/acceptance/bundle/state/engine_default/script +++ b/acceptance/bundle/state/engine_default/script @@ -1,4 +1,4 @@ -title 'No engine configured and no state: deploy defaults to the direct engine\n' +title 'No engine configured and no state: deploy defaults to the direct engine' unset DATABRICKS_BUNDLE_ENGINE trace $CLI bundle deploy trace print_requests.py //api/2.1/unity-catalog/schemas | jq '.headers["User-Agent"][0]' | contains.py 'engine/direct' '!terraform' '!tf-provider' diff --git a/acceptance/bundle/state/engine_mismatch/script b/acceptance/bundle/state/engine_mismatch/script index 450ef1835eb..55b6276d5be 100644 --- a/acceptance/bundle/state/engine_mismatch/script +++ b/acceptance/bundle/state/engine_mismatch/script @@ -3,9 +3,9 @@ echo '{"serial": 1, "lineage": "test-lineage"}' > .databricks/bundle/default/ter # `bundle debug states` doesn't deploy; it should just report the state without # a mismatch warning. The "will attempt to migrate" hint belongs to deploy only. -title 'Direct opt-in with terraform state on disk: debug states reports without warning\n' +title 'Direct opt-in with terraform state on disk: debug states reports without warning' unset DATABRICKS_BUNDLE_ENGINE trace $CLI bundle debug states -title 'Invalid env var with config set: config takes priority, still no warning\n' +title 'Invalid env var with config set: config takes priority, still no warning' trace DATABRICKS_BUNDLE_ENGINE=bla $CLI bundle debug states diff --git a/acceptance/bundle/state/force_pull_commands/output.txt b/acceptance/bundle/state/force_pull_commands/output.txt index 9f5d004ce19..307e5d54bce 100644 --- a/acceptance/bundle/state/force_pull_commands/output.txt +++ b/acceptance/bundle/state/force_pull_commands/output.txt @@ -6,6 +6,7 @@ Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Use a fake browser that just prints the URL it would have opened + === bundle summary without --force-pull: no remote state read >>> [CLI] bundle summary diff --git a/acceptance/bundle/state/force_pull_commands/script b/acceptance/bundle/state/force_pull_commands/script index 19b0748d61e..f1677be5c9b 100644 --- a/acceptance/bundle/state/force_pull_commands/script +++ b/acceptance/bundle/state/force_pull_commands/script @@ -9,22 +9,22 @@ export BROWSER="echo_browser.py" # --force-pull make zero HTTP requests, so the file is never created and # print_requests.py would otherwise exit with "File not found". -title "bundle summary without --force-pull: no remote state read\n" +title "bundle summary without --force-pull: no remote state read" trace $CLI bundle summary > /dev/null touch "$OUT_REQUESTS" print_requests.py --get //workspace/export -title "bundle summary --force-pull: remote state read\n" +title "bundle summary --force-pull: remote state read" trace $CLI bundle summary --force-pull > /dev/null touch "$OUT_REQUESTS" print_requests.py --get //workspace/export -title "bundle open without --force-pull: no remote state read\n" +title "bundle open without --force-pull: no remote state read" trace $CLI bundle open foo > /dev/null touch "$OUT_REQUESTS" print_requests.py --get //workspace/export -title "bundle open --force-pull: remote state read\n" +title "bundle open --force-pull: remote state read" trace $CLI bundle open foo --force-pull > /dev/null touch "$OUT_REQUESTS" print_requests.py --get //workspace/export diff --git a/acceptance/bundle/state/newer_cli_version/script b/acceptance/bundle/state/newer_cli_version/script index 172dc126cb3..c21b7215c88 100644 --- a/acceptance/bundle/state/newer_cli_version/script +++ b/acceptance/bundle/state/newer_cli_version/script @@ -1,9 +1,9 @@ -title "State written by a newer CLI: warn, but proceed\n" +title "State written by a newer CLI: warn, but proceed" mkdir -p .databricks/bundle/default cp resources.newer.json .databricks/bundle/default/resources.json trace $CLI bundle plan -title "State written by this CLI: no warning\n" +title "State written by this CLI: no warning" # The running version is only known at test time, so derive the state's # cli_version from the binary itself rather than committing it as a fixture. version=$($CLI version | sed 's/^Databricks CLI v//') diff --git a/acceptance/bundle/state/permission_level_migration/output.txt b/acceptance/bundle/state/permission_level_migration/output.txt index 2091843525f..5ad53b46e07 100644 --- a/acceptance/bundle/state/permission_level_migration/output.txt +++ b/acceptance/bundle/state/permission_level_migration/output.txt @@ -1,8 +1,10 @@ === Plan with old v1 state + >>> [CLI] bundle plan -o json === Deploy (migrates state) + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created jobs.my_job @@ -11,6 +13,7 @@ Files: 3 uploaded, 0 deleted Resources: 1 created, 1 changed, 0 deleted, 0 unchanged === Print state after deploy + >>> print_state.py { "state_version": 3, diff --git a/acceptance/bundle/state/same_serial/output.txt b/acceptance/bundle/state/same_serial/output.txt index 93a58414de8..3a8ef910436 100644 --- a/acceptance/bundle/state/same_serial/output.txt +++ b/acceptance/bundle/state/same_serial/output.txt @@ -1,5 +1,6 @@ === If serial is the same, refuse to proceed + >>> musterr [CLI] bundle deploy Error: same serial number in terraform and direct states diff --git a/acceptance/bundle/state/state_present/output.txt b/acceptance/bundle/state/state_present/output.txt index a2e6d0b618a..24ac39aae47 100644 --- a/acceptance/bundle/state/state_present/output.txt +++ b/acceptance/bundle/state/state_present/output.txt @@ -13,6 +13,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "cli/[CLI_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_plan cmd-exec-id/[UUID][0] interactive/none lineage/ARgQbIVroS5 engine/direct auth/pat" === A resources.json with a lower serial does not win: the terraform state still migrates + >>> [CLI] bundle plan Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/state/state_present/script b/acceptance/bundle/state/state_present/script index 40d2f72e440..22122d53b07 100644 --- a/acceptance/bundle/state/state_present/script +++ b/acceptance/bundle/state/state_present/script @@ -1,4 +1,4 @@ -title 'Terraform state present: the direct default migrates it to the direct engine\n' +title 'Terraform state present: the direct default migrates it to the direct engine' # Keep the committed fixtures (and generated files) out of `bundle deploy`'s upload. printf '%s\n' '.gitignore' 'out.*' 'terraform*.tfstate' 'terraform*-requests.json' > .gitignore @@ -21,7 +21,7 @@ echo "{\"serial\": 0, \"lineage\": \"$lineage\"}" > .databricks/bundle/default/r trace $CLI bundle plan rm "$OUT_REQUESTS" -title 'A resources.json with a higher serial wins: the direct state is authoritative, no migration\n' +title 'A resources.json with a higher serial wins: the direct state is authoritative, no migration' echo "{\"serial\": 10, \"lineage\": \"$lineage\"}" > .databricks/bundle/default/resources.json # serial 10 outranks the terraform state (serial 1), so direct wins; its (empty) state plans to # create the schema, and every schemas request carries the direct-engine User-Agent, not terraform. diff --git a/acceptance/bundle/strict_profile_match/config_file/output.txt b/acceptance/bundle/strict_profile_match/config_file/output.txt index 6814e82e21d..aff3369bf64 100644 --- a/acceptance/bundle/strict_profile_match/config_file/output.txt +++ b/acceptance/bundle/strict_profile_match/config_file/output.txt @@ -1,5 +1,6 @@ === A target that matches several profiles, including [DEFAULT], is rejected + >>> errcode env DATABRICKS_CONFIG_FILE=[TEST_TMP_DIR]/ambiguous.databrickscfg [CLI] bundle validate -t classic Error: cannot resolve bundle auth configuration: resolve: strict profile matching: multiple profiles match host [DATABRICKS_URL]: DEFAULT, workspace_classic. Config: host=[DATABRICKS_URL], config_file=[TEST_TMP_DIR]/ambiguous.databrickscfg, databricks_cli_path=[CLI]. Env: DATABRICKS_CONFIG_FILE, DATABRICKS_CLI_PATH @@ -13,6 +14,7 @@ Found 1 error Exit code: 1 === Account profiles are skipped + >>> errcode env DATABRICKS_CONFIG_FILE=[TEST_TMP_DIR]/account.databrickscfg [CLI] bundle run -t custom_host show_auth profile=workspace_custom_host host=[CUSTOM_HOST] workspace_id=[WORKSPACE_ID] @@ -39,6 +41,7 @@ Found 1 error Exit code: 1 === A missing config file is rejected + >>> errcode env DATABRICKS_CONFIG_FILE=[TEST_TMP_DIR]/missing.databrickscfg [CLI] bundle validate -t classic Error: cannot resolve bundle auth configuration: resolve: strict profile matching: config file [TEST_TMP_DIR]/missing.databrickscfg not found. Config: host=[DATABRICKS_URL], config_file=[TEST_TMP_DIR]/missing.databrickscfg, databricks_cli_path=[CLI]. Env: DATABRICKS_CONFIG_FILE, DATABRICKS_CLI_PATH diff --git a/acceptance/bundle/strict_profile_match/default_profile/host/output.txt b/acceptance/bundle/strict_profile_match/default_profile/host/output.txt index 2d194ce17e2..e75025f18d8 100644 --- a/acceptance/bundle/strict_profile_match/default_profile/host/output.txt +++ b/acceptance/bundle/strict_profile_match/default_profile/host/output.txt @@ -1,17 +1,21 @@ === Target classic matches by host, not the default profile + >>> errcode [CLI] bundle run -t classic show_auth profile=workspace_classic host=[DATABRICKS_URL] workspace_id=[WORKSPACE_ID] === Target classic_with_workspace_id matches by host, not the default profile + >>> errcode [CLI] bundle run -t classic_with_workspace_id show_auth profile=workspace_classic_id host=[DATABRICKS_URL] workspace_id=[WORKSPACE_ID] === Target custom_host matches by host, not the default profile + >>> errcode [CLI] bundle run -t custom_host show_auth profile=workspace_custom_host host=[CUSTOM_HOST] workspace_id=[WORKSPACE_ID] === Target classic_other_workspace is rejected + >>> errcode [CLI] bundle validate -t classic_other_workspace Error: cannot resolve bundle auth configuration: resolve: strict profile matching: no profile for host [DATABRICKS_URL] has workspace_id 2222222222. Config: host=[DATABRICKS_URL], workspace_id=2222222222, config_file=[TEST_TMP_DIR]/.databrickscfg, databricks_cli_path=[CLI]. Env: DATABRICKS_CONFIG_FILE, DATABRICKS_CLI_PATH @@ -25,6 +29,7 @@ Found 1 error Exit code: 1 === Target custom_host_without_workspace_id is rejected + >>> errcode [CLI] bundle validate -t custom_host_without_workspace_id Error: cannot resolve bundle auth configuration: resolve: strict profile matching: host [CUSTOM_HOST] requires a workspace ID; set workspace.workspace_id or add ?o= to workspace.host. Config: host=[CUSTOM_HOST], config_file=[TEST_TMP_DIR]/.databrickscfg, databricks_cli_path=[CLI]. Env: DATABRICKS_CONFIG_FILE, DATABRICKS_CLI_PATH diff --git a/acceptance/bundle/strict_profile_match/default_profile/no_host/output.txt b/acceptance/bundle/strict_profile_match/default_profile/no_host/output.txt index 614d59ce296..e3e6f453ed8 100644 --- a/acceptance/bundle/strict_profile_match/default_profile/no_host/output.txt +++ b/acceptance/bundle/strict_profile_match/default_profile/no_host/output.txt @@ -1,13 +1,16 @@ === Target no_host uses the default profile + >>> errcode [CLI] bundle run -t no_host show_auth profile=workspace_classic_id host=[DATABRICKS_URL] workspace_id=[WORKSPACE_ID] === Target no_host_with_workspace_id uses the default profile + >>> errcode [CLI] bundle run -t no_host_with_workspace_id show_auth profile=workspace_classic_id host=[DATABRICKS_URL] workspace_id=[WORKSPACE_ID] === Target no_host_other_workspace is rejected + >>> errcode [CLI] bundle validate -t no_host_other_workspace Error: cannot resolve bundle auth configuration: resolve: strict profile matching: profile workspace_classic_id does not have workspace_id 2222222222. Config: workspace_id=2222222222, profile=workspace_classic_id, config_file=[TEST_TMP_DIR]/.databrickscfg, databricks_cli_path=[CLI]. Env: DATABRICKS_CONFIG_FILE, DATABRICKS_CLI_PATH diff --git a/acceptance/bundle/strict_profile_match/disabled/output.txt b/acceptance/bundle/strict_profile_match/disabled/output.txt index 27c5f5f642b..3fb4585dcbf 100644 --- a/acceptance/bundle/strict_profile_match/disabled/output.txt +++ b/acceptance/bundle/strict_profile_match/disabled/output.txt @@ -1,5 +1,6 @@ === Without strict mode, the environment token is used for a custom host of another workspace + >>> errcode [CLI] bundle validate -t custom_host_other_workspace Name: strict-profile-match Target: custom_host_other_workspace @@ -11,6 +12,7 @@ Workspace: Validation OK! === The variable is read as a boolean, so false leaves strict mode off + >>> errcode env DATABRICKS_BUNDLE_STRICT_PROFILE_MATCH=false [CLI] bundle validate -t custom_host_other_workspace Name: strict-profile-match Target: custom_host_other_workspace diff --git a/acceptance/bundle/strict_profile_match/host/output.txt b/acceptance/bundle/strict_profile_match/host/output.txt index 6c3dcbeefab..61f7b75f9d6 100644 --- a/acceptance/bundle/strict_profile_match/host/output.txt +++ b/acceptance/bundle/strict_profile_match/host/output.txt @@ -1,21 +1,26 @@ === Target classic matches one profile + >>> errcode [CLI] bundle run -t classic show_auth profile=workspace_classic host=[DATABRICKS_URL] workspace_id=[WORKSPACE_ID] === Target classic_with_workspace_id matches one profile + >>> errcode [CLI] bundle run -t classic_with_workspace_id show_auth profile=workspace_classic_id host=[DATABRICKS_URL] workspace_id=[WORKSPACE_ID] === Target custom_host matches one profile + >>> errcode [CLI] bundle run -t custom_host show_auth profile=workspace_custom_host host=[CUSTOM_HOST] workspace_id=[WORKSPACE_ID] === Target custom_host_workspace_id_field matches one profile + >>> errcode [CLI] bundle run -t custom_host_workspace_id_field show_auth profile=workspace_custom_host host=[CUSTOM_HOST] workspace_id=[WORKSPACE_ID] === Target classic_other_workspace is rejected + >>> errcode [CLI] bundle validate -t classic_other_workspace Error: cannot resolve bundle auth configuration: resolve: strict profile matching: no profile for host [DATABRICKS_URL] has workspace_id 2222222222. Config: host=[DATABRICKS_URL], workspace_id=2222222222, config_file=[TEST_TMP_DIR]/.databrickscfg, databricks_cli_path=[CLI]. Env: DATABRICKS_CONFIG_FILE, DATABRICKS_CLI_PATH @@ -29,6 +34,7 @@ Found 1 error Exit code: 1 === Target custom_host_without_workspace_id is rejected + >>> errcode [CLI] bundle validate -t custom_host_without_workspace_id Error: cannot resolve bundle auth configuration: resolve: strict profile matching: host [CUSTOM_HOST] requires a workspace ID; set workspace.workspace_id or add ?o= to workspace.host. Config: host=[CUSTOM_HOST], config_file=[TEST_TMP_DIR]/.databrickscfg, databricks_cli_path=[CLI]. Env: DATABRICKS_CONFIG_FILE, DATABRICKS_CLI_PATH @@ -42,6 +48,7 @@ Found 1 error Exit code: 1 === Target custom_host_other_workspace is rejected + >>> errcode [CLI] bundle validate -t custom_host_other_workspace Error: cannot resolve bundle auth configuration: resolve: strict profile matching: no profile for host [CUSTOM_HOST] has workspace_id 2222222222. Config: host=[CUSTOM_HOST], workspace_id=2222222222, config_file=[TEST_TMP_DIR]/.databrickscfg, databricks_cli_path=[CLI]. Env: DATABRICKS_CONFIG_FILE, DATABRICKS_CLI_PATH @@ -55,6 +62,7 @@ Found 1 error Exit code: 1 === Target unknown_host is rejected + >>> errcode [CLI] bundle validate -t unknown_host Error: cannot resolve bundle auth configuration: resolve: strict profile matching: no workspace profile matches host https://unknown.test. Config: host=https://unknown.test, config_file=[TEST_TMP_DIR]/.databrickscfg, databricks_cli_path=[CLI]. Env: DATABRICKS_CONFIG_FILE, DATABRICKS_CLI_PATH diff --git a/acceptance/bundle/strict_profile_match/list_targets/output.txt b/acceptance/bundle/strict_profile_match/list_targets/output.txt index 03af2d917f0..404c6552829 100644 --- a/acceptance/bundle/strict_profile_match/list_targets/output.txt +++ b/acceptance/bundle/strict_profile_match/list_targets/output.txt @@ -1,5 +1,6 @@ === list-targets reports the inherited workspace ID + >>> [CLI] bundle debug list-targets -o json { "targets": [ @@ -12,5 +13,6 @@ ] } === Authentication uses the same workspace ID + >>> errcode [CLI] bundle run -t inherited_workspace_id show_auth profile=workspace_custom_host host=[CUSTOM_HOST] workspace_id=[WORKSPACE_ID] diff --git a/acceptance/bundle/strict_profile_match/no_host/output.txt b/acceptance/bundle/strict_profile_match/no_host/output.txt index 087e9743cb1..06b80530865 100644 --- a/acceptance/bundle/strict_profile_match/no_host/output.txt +++ b/acceptance/bundle/strict_profile_match/no_host/output.txt @@ -1,25 +1,31 @@ === A target without a host, workspace ID or profile resolves as without strict mode + >>> errcode env DATABRICKS_HOST=[DATABRICKS_URL] [CLI] bundle run -t no_host show_auth profile= host=[DATABRICKS_URL] workspace_id=[WORKSPACE_ID] === A target without a host uses the explicit profile + >>> errcode [CLI] bundle run -t no_host -p workspace_custom_host show_auth profile=workspace_custom_host host=[CUSTOM_HOST] workspace_id=[WORKSPACE_ID] === A target without a host uses workspace.profile + >>> errcode [CLI] bundle run -t no_host_with_profile show_auth profile=workspace_custom_host host=[CUSTOM_HOST] workspace_id=[WORKSPACE_ID] === A target without a host uses the default profile + >>> errcode env DATABRICKS_CONFIG_FILE=[TEST_TMP_DIR]/default.databrickscfg [CLI] bundle run -t no_host show_auth profile=workspace_classic host=[DATABRICKS_URL] workspace_id=[WORKSPACE_ID] === A target without a host accepts a profile with the target's workspace ID + >>> errcode [CLI] bundle run -t no_host_with_workspace_id -p workspace_classic_id show_auth profile=workspace_classic_id host=[DATABRICKS_URL] workspace_id=[WORKSPACE_ID] === A target without a host rejects a profile without the target's workspace ID + >>> errcode env DATABRICKS_CONFIG_FILE=[TEST_TMP_DIR]/default.databrickscfg [CLI] bundle validate -t no_host_with_workspace_id Error: cannot resolve bundle auth configuration: resolve: strict profile matching: profile workspace_classic does not have workspace_id [WORKSPACE_ID]. Config: workspace_id=[WORKSPACE_ID], profile=workspace_classic, config_file=[TEST_TMP_DIR]/default.databrickscfg, databricks_cli_path=[CLI]. Env: DATABRICKS_CONFIG_FILE, DATABRICKS_CLI_PATH @@ -31,6 +37,7 @@ Found 1 error Exit code: 1 === A target without a host rejects an unknown profile + >>> errcode [CLI] bundle validate -t no_host -p my-local-profile Error: cannot resolve bundle auth configuration: resolve: [TEST_TMP_DIR]/.databrickscfg has no my-local-profile profile configured. Config: profile=my-local-profile, config_file=[TEST_TMP_DIR]/.databrickscfg, databricks_cli_path=[CLI]. Env: DATABRICKS_CONFIG_FILE, DATABRICKS_CLI_PATH diff --git a/acceptance/bundle/strict_profile_match/profile/output.txt b/acceptance/bundle/strict_profile_match/profile/output.txt index f73f8a1aa10..8ea1dbe21f4 100644 --- a/acceptance/bundle/strict_profile_match/profile/output.txt +++ b/acceptance/bundle/strict_profile_match/profile/output.txt @@ -1,9 +1,11 @@ === A workspace.profile that matches the target is accepted + >>> errcode [CLI] bundle run -t classic_with_matching_profile show_auth profile=workspace_classic host=[DATABRICKS_URL] workspace_id=[WORKSPACE_ID] === A workspace.profile that doesn't match the target is rejected + >>> errcode [CLI] bundle validate -t classic_with_profile Error: cannot resolve bundle auth configuration: resolve: strict profile matching: profile my-local-profile does not match profile workspace_classic selected for host [DATABRICKS_URL]. Config: host=[DATABRICKS_URL], profile=my-local-profile, config_file=[TEST_TMP_DIR]/.databrickscfg, databricks_cli_path=[CLI]. Env: DATABRICKS_CONFIG_FILE, DATABRICKS_CLI_PATH @@ -17,10 +19,12 @@ Found 1 error Exit code: 1 === An explicit profile that matches the target is accepted + >>> errcode [CLI] bundle run -t classic -p workspace_classic show_auth profile=workspace_classic host=[DATABRICKS_URL] workspace_id=[WORKSPACE_ID] === An explicit profile that doesn't match the target is rejected + >>> errcode [CLI] bundle validate -t classic -p workspace_classic_id Error: cannot resolve bundle auth configuration: resolve: strict profile matching: profile workspace_classic_id does not match profile workspace_classic selected for host [DATABRICKS_URL]. Config: host=[DATABRICKS_URL], profile=workspace_classic_id, config_file=[TEST_TMP_DIR]/.databrickscfg, databricks_cli_path=[CLI]. Env: DATABRICKS_CONFIG_FILE, DATABRICKS_CLI_PATH @@ -34,9 +38,11 @@ Found 1 error Exit code: 1 === An explicit profile wins over workspace.profile, as without strict mode + >>> errcode [CLI] bundle run -t classic_with_profile -p workspace_classic show_auth profile=workspace_classic host=[DATABRICKS_URL] workspace_id=[WORKSPACE_ID] === A CLI started from a bundle script uses the selected profile + >>> errcode [CLI] bundle run -t custom_host nested_validate workspace_custom_host diff --git a/acceptance/bundle/summary/missing-libraries-file-path/output.txt b/acceptance/bundle/summary/missing-libraries-file-path/output.txt index 01d9155caff..d3fa257cb04 100644 --- a/acceptance/bundle/summary/missing-libraries-file-path/output.txt +++ b/acceptance/bundle/summary/missing-libraries-file-path/output.txt @@ -1,5 +1,6 @@ === Initial view of resources without id and modified_status=created + >>> [CLI] bundle summary -o json Error: file intentionally-missing.py not found diff --git a/acceptance/bundle/summary/modified_status/output.txt b/acceptance/bundle/summary/modified_status/output.txt index b64c1a66aaa..21b476ba55f 100644 --- a/acceptance/bundle/summary/modified_status/output.txt +++ b/acceptance/bundle/summary/modified_status/output.txt @@ -1,5 +1,6 @@ === Initial view of resources without id and modified_status=created + >>> [CLI] bundle summary -o json { "alerts": { @@ -77,6 +78,7 @@ Files: 4 uploaded, 0 deleted Resources: 4 created, 0 changed, 0 deleted, 0 unchanged === Post-deployment view of resources with id and without modified_status + >>> [CLI] bundle summary -o json { "alerts": { @@ -149,6 +151,7 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged } === Expecting all resources to have modified_status=deleted + >>> [CLI] bundle summary -o json { "alerts": { diff --git a/acceptance/bundle/sync/replace-node-type/script b/acceptance/bundle/sync/replace-node-type/script index eca6be849c3..0e379d0ca07 100644 --- a/acceptance/bundle/sync/replace-node-type/script +++ b/acceptance/bundle/sync/replace-node-type/script @@ -14,17 +14,17 @@ list() { $CLI workspace list -o json "$remote" | jq -r 'sort_by(.path)[] | "\(.object_type) \(.path | sub(".*/"; ""))"' } -title "A file holds the name of notebook bom.sql, a notebook holds the name of file plain.sql\n" +title "A file holds the name of notebook bom.sql, a notebook holds the name of file plain.sql" $CLI workspace mkdirs "$remote" trace $CLI workspace import "$remote/bom.sql" --file bom.sql --format RAW trace $CLI workspace import "$remote/plain" --file plain.sql --format SOURCE --language SQL trace list -title "Sync replaces both\n" +title "Sync replaces both" trace $CLI bundle sync --output text 2>&1 | sort trace list -title "Remove both local files and sync again\n" +title "Remove both local files and sync again" rm bom.sql plain.sql trace $CLI bundle sync --output text 2>&1 | sort trace list diff --git a/acceptance/bundle/sync/zip-extract/script b/acceptance/bundle/sync/zip-extract/script index 16f6d6df107..11c47731957 100644 --- a/acceptance/bundle/sync/zip-extract/script +++ b/acceptance/bundle/sync/zip-extract/script @@ -13,12 +13,12 @@ list() { $CLI workspace list -o json "$1" | jq -r 'sort_by(.path)[] | "\(.object_type) \(.path | sub(".*/files"; ""))"' } -title "Sync extracts the archive into the bundle's file path\n" +title "Sync extracts the archive into the bundle's file path" trace $CLI bundle sync trace list "$remote" trace list "$remote/pkg" -title "Syncing a changed archive overwrites the extracted files\n" +title "Syncing a changed archive overwrites the extracted files" cp archive_v2.zip archive.zip trace $CLI bundle sync trace $CLI workspace export "$remote/inner.txt" --format AUTO diff --git a/acceptance/bundle/telemetry/deploy-artifacts-variables/output.txt b/acceptance/bundle/telemetry/deploy-artifacts-variables/output.txt index ba5bba6a5ab..75dde3d31b6 100644 --- a/acceptance/bundle/telemetry/deploy-artifacts-variables/output.txt +++ b/acceptance/bundle/telemetry/deploy-artifacts-variables/output.txt @@ -8,6 +8,7 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === No artifact metrics expected + >>> cat [OUTPUT_DIR]/out.requests.txt >>> [CLI] bundle deploy -t prod @@ -19,6 +20,7 @@ Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Artifact metrics expected + >>> cat [OUTPUT_DIR]/out.requests.txt { "key": "artifacts_reference_used", diff --git a/acceptance/bundle/telemetry/deploy-immutable-snapshot/script b/acceptance/bundle/telemetry/deploy-immutable-snapshot/script index ab610421285..a08dfa8b65b 100644 --- a/acceptance/bundle/telemetry/deploy-immutable-snapshot/script +++ b/acceptance/bundle/telemetry/deploy-immutable-snapshot/script @@ -4,13 +4,13 @@ print_snapshot_upload_duration_included() { jq -r 'select(.path? == "/telemetry-ext") | (.body.protoLogs // [])[] | fromjson | (.entry // .) | .databricks_cli_log.bundle_deploy_event.experimental.bundle_mutator_execution_time_ms // [] | any(.key == "snapshot.upload" and .value >= 0)' "$OUT_REQUESTS" } -title "First deploy uploads the snapshot\n" +title "First deploy uploads the snapshot" trace $CLI bundle deploy trace print_telemetry_bool_values | grep immutable_snapshot_used trace print_snapshot_upload_duration_included rm "$OUT_REQUESTS" -title "Second deploy finds the snapshot already uploaded\n" +title "Second deploy finds the snapshot already uploaded" trace $CLI bundle deploy trace print_telemetry_bool_values | grep immutable_snapshot_used trace print_snapshot_upload_duration_included diff --git a/acceptance/bundle/telemetry/deploy/output.txt b/acceptance/bundle/telemetry/deploy/output.txt index f442634c52b..2953d55e717 100644 --- a/acceptance/bundle/telemetry/deploy/output.txt +++ b/acceptance/bundle/telemetry/deploy/output.txt @@ -12,7 +12,8 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged >>> cat [OUTPUT_DIR]/out.requests.txt === Assert that mutator execution times are being recorded + >>> cat telemetry.json true -=== Assert the terraform engine opt-in is reported \ No newline at end of file +=== Assert the terraform engine opt-in is reported diff --git a/acceptance/bundle/templates-machinery/supported-url/output.txt b/acceptance/bundle/templates-machinery/supported-url/output.txt index 959b4c6fb82..29dd76a8162 100644 --- a/acceptance/bundle/templates-machinery/supported-url/output.txt +++ b/acceptance/bundle/templates-machinery/supported-url/output.txt @@ -1,4 +1,6 @@ === https:// URL is recognized as a Git URL + === ssh:// URL is recognized as a Git URL -=== git@ URL is recognized as a Git URL \ No newline at end of file + +=== git@ URL is recognized as a Git URL diff --git a/acceptance/bundle/templates-machinery/workspace_variable/input/script b/acceptance/bundle/templates-machinery/workspace_variable/input/script index 8d7ec6d1ef1..432f058c5f2 100644 --- a/acceptance/bundle/templates-machinery/workspace_variable/input/script +++ b/acceptance/bundle/templates-machinery/workspace_variable/input/script @@ -2,7 +2,7 @@ # context, with and without a value supplied. export DATABRICKS_CONFIG_PROFILE=workspace_custom_host -title "String input\n" +title "String input" trace $CLI bundle debug render-template-schema ./string | schema_fields trace $CLI bundle debug render-template-schema ./string --input-file input.value.json | schema_fields for config in "" "--config-file input.empty.json" "--config-file input.value.json"; do @@ -14,21 +14,21 @@ trace musterr $CLI bundle init ./string --config-file input.invalid.json # The welcome message is rendered before inputs get their values. The input # still owns the name, so the workspace context isn't used in its place. -title "Input without a value yet\n" +title "Input without a value yet" trace $CLI bundle debug render-template-schema ./no_value | schema_fields trace musterr $CLI bundle init ./no_value trace $CLI bundle init ./no_value --config-file input.value.json cat workspace.txt rm workspace.txt -title "Boolean input\n" +title "Boolean input" for config in "" "--config-file input.false.json"; do trace $CLI bundle init ./boolean $config cat workspace.txt rm workspace.txt done -title "Integer input\n" +title "Integer input" for config in "" "--config-file input.zero.json"; do trace $CLI bundle init ./integer $config cat workspace.txt @@ -37,7 +37,7 @@ done # An input value that the schema doesn't declare doesn't claim the name, so # schema rendering and bundle init both use the workspace context. -title "Undeclared input\n" +title "Undeclared input" trace $CLI bundle debug render-template-schema ./undeclared --input-file input.value.json | schema_fields trace $CLI bundle init ./undeclared --config-file input.value.json cat workspace.txt diff --git a/acceptance/bundle/templates-machinery/workspace_variable/runtime/script b/acceptance/bundle/templates-machinery/workspace_variable/runtime/script index c670d6dcec1..2651beb8ad5 100644 --- a/acceptance/bundle/templates-machinery/workspace_variable/runtime/script +++ b/acceptance/bundle/templates-machinery/workspace_variable/runtime/script @@ -1,5 +1,5 @@ for profile in workspace_classic workspace_classic_id workspace_custom_host no_workspace_id; do - title "Profile $profile\n" + title "Profile $profile" trace $CLI bundle debug render-template-schema ./urls --profile $profile | schema_fields trace $CLI bundle init ./urls --profile $profile cat urls.txt @@ -9,7 +9,7 @@ done # Render the schema again with the current inputs, then run init with the same # inputs and profile. The input has an edited jobs_url: the schema still shows # its rendered default, and init keeps the edited value. -title "Rerender and init with edited inputs\n" +title "Rerender and init with edited inputs" trace $CLI bundle debug render-template-schema ./urls --profile workspace_custom_host --input-file input.edited.json | schema_fields trace $CLI bundle init ./urls --profile workspace_custom_host --config-file input.edited.json cat urls.txt @@ -18,26 +18,26 @@ rm urls.txt # A default that fails, here because the profile has no workspace ID, doesn't # matter when the input has a value: init doesn't use it, and the schema keeps # it as written. Without a value, the schema still fails. -title "Failing default of an input with a value\n" +title "Failing default of an input with a value" trace $CLI bundle debug render-template-schema ./supplied --profile no_workspace_id --input-file input.supplied.json | schema_fields trace $CLI bundle init ./supplied --profile no_workspace_id --config-file input.supplied.json cat value.txt rm value.txt trace musterr $CLI bundle debug render-template-schema ./supplied --profile no_workspace_id -title "Invalid URL paths\n" +title "Invalid URL paths" for input in absolute-url protocol-relative workspace-selector selector-w selector-workspace-id semicolon; do trace musterr $CLI bundle init ./urls --profile workspace_custom_host --config-file input.$input.json done # No profile, so that only the environment variable sets the ID. -title "Workspace ID from environment variables\n" +title "Workspace ID from environment variables" trace DATABRICKS_CONFIG_FILE=/dev/null DATABRICKS_HOST=$CUSTOM_HOST DATABRICKS_WORKSPACE_ID=$WORKSPACE_ID $CLI bundle init ./urls grep ^URL urls.txt rm urls.txt # bundle init reads the workspace configuration of a bundle in the working directory. -title "Workspace ID from the bundle in the working directory\n" +title "Workspace ID from the bundle in the working directory" mkdir in_bundle cat > in_bundle/databricks.yml <>> musterr [CLI] bundle init app-appkit Error: app-appkit requires project_name in --config-file in non-interactive mode === Unsupported template directory fails before initialization + >>> musterr [CLI] bundle init app-appkit --template-dir template Error: --template-dir is not supported for app-appkit; use databricks apps init --template for a custom AppKit template === Version and branch cannot be combined + >>> musterr [CLI] bundle init app-appkit --config-file input-version.json --branch main Error: version in --config-file cannot be combined with --tag or --branch === JSON output is unsupported + >>> musterr [CLI] bundle init app-appkit --config-file input.json -o json Error: -o json is not supported for app-appkit; use text output === Initialize AppKit from a config file without a terminal + >>> [CLI] bundle init app-appkit --config-file input.json --output-dir output ✔ Creating project diff --git a/acceptance/bundle/templates/dbt-sql-workspace-id/script b/acceptance/bundle/templates/dbt-sql-workspace-id/script index f7cfa1bd350..86e862cce37 100644 --- a/acceptance/bundle/templates/dbt-sql-workspace-id/script +++ b/acceptance/bundle/templates/dbt-sql-workspace-id/script @@ -17,7 +17,7 @@ unset DATABRICKS_TOKEN # dbt reads the workspace ID from ?o= in the HTTP path. The template adds it # when the profile sets the ID, like ?w= in the bundle host. for profile in no_workspace_id custom_host; do - title "Profile $profile\n" + title "Profile $profile" trace $CLI bundle init dbt-sql --config-file ./input.json --profile $profile trace grep 'host:' my_dbt_sql/databricks.yml trace grep -E "^ default: [^{]" my_dbt_sql/profile_template.yml diff --git a/acceptance/bundle/templates/default-minimal/profiles/script b/acceptance/bundle/templates/default-minimal/profiles/script index f3d1d94a153..a42a9d9659f 100644 --- a/acceptance/bundle/templates/default-minimal/profiles/script +++ b/acceptance/bundle/templates/default-minimal/profiles/script @@ -18,7 +18,7 @@ unset DATABRICKS_HOST unset DATABRICKS_TOKEN for profile in classic custom_host; do - title "Profile $profile\n" + title "Profile $profile" trace $CLI bundle init default-minimal --config-file ./input.json --profile $profile cd my_default_minimal trace grep 'host:' databricks.yml diff --git a/acceptance/bundle/validate/compute_id_location/output.txt b/acceptance/bundle/validate/compute_id_location/output.txt index 559b500c30a..fc0638f2a56 100644 --- a/acceptance/bundle/validate/compute_id_location/output.txt +++ b/acceptance/bundle/validate/compute_id_location/output.txt @@ -1,5 +1,6 @@ === default target without compute_id + >>> [CLI] bundle validate Warning: compute_id is deprecated, please use cluster_id instead at bundle.compute_id @@ -18,6 +19,7 @@ Workspace: Found 2 warnings === target with compute_id + >>> [CLI] bundle validate -t dev Warning: compute_id is deprecated, please use cluster_id instead at bundle.compute_id diff --git a/acceptance/bundle/validate/override_effective_location/output.txt b/acceptance/bundle/validate/override_effective_location/output.txt index e3fa5193367..298ac4093ac 100644 --- a/acceptance/bundle/validate/override_effective_location/output.txt +++ b/acceptance/bundle/validate/override_effective_location/output.txt @@ -1,5 +1,6 @@ === engine overridden in target + >>> [CLI] bundle validate -t bad_engine Error: invalid value "invalid_value" for bundle.engine (expected "direct") in databricks.yml:12:15 @@ -12,6 +13,7 @@ Found 1 error Exit code: 1 === trigger_pause_status overridden in development target + >>> [CLI] bundle validate -t dev_trigger Error: target with 'mode: development' cannot set trigger pause status to UNPAUSED by default in databricks.yml:17:29 @@ -27,6 +29,7 @@ Found 1 error Exit code: 1 === name_prefix overridden in development target + >>> [CLI] bundle validate -t dev_prefix Error: prefix should contain the current username or ${workspace.current_user.short_name} to ensure uniqueness when using 'mode: development' in databricks.yml:21:20 diff --git a/acceptance/bundle/validate/presets_name_prefix_dev/script b/acceptance/bundle/validate/presets_name_prefix_dev/script index f18f5384f7c..c3809f3badf 100644 --- a/acceptance/bundle/validate/presets_name_prefix_dev/script +++ b/acceptance/bundle/validate/presets_name_prefix_dev/script @@ -1,22 +1,22 @@ -title "Remove presets: section entirely - out.without_presets.json\n" +title "Remove presets: section entirely - out.without_presets.json" grep -v PRESETS databricks.yml.tmpl > databricks.yml trace musterr grep -C 1 prefix: databricks.yml $CLI bundle validate -o json > out.without_presets.json -title "Set name_prefix="prefix" - out.with_static_prefix.json\n" +title "Set name_prefix="prefix" - out.with_static_prefix.json" PREFIX="prefix" envsubst < databricks.yml.tmpl > databricks.yml trace grep -C 1 prefix: databricks.yml trace musterr $CLI bundle validate -o json > out.with_static_prefix.json -title "Set name_prefix="" - out.with_empty_prefix.json\n" +title "Set name_prefix="" - out.with_empty_prefix.json" PREFIX='""' envsubst < databricks.yml.tmpl > databricks.yml trace grep -C 1 prefix: databricks.yml trace $CLI bundle validate -o json > out.with_empty_prefix.json -title "Set name_prefix= - out.with_nil_prefix.json\n" +title "Set name_prefix= - out.with_nil_prefix.json" PREFIX= envsubst < databricks.yml.tmpl > databricks.yml trace grep -C 1 prefix: databricks.yml trace $CLI bundle validate -o json > out.with_nil_prefix.json rm databricks.yml -title "The end\n" # prevent dangling empty line +title "The end" diff --git a/acceptance/bundle/validate/sync_mode/output.txt b/acceptance/bundle/validate/sync_mode/output.txt index 62860acdb96..ee2483ffb85 100644 --- a/acceptance/bundle/validate/sync_mode/output.txt +++ b/acceptance/bundle/validate/sync_mode/output.txt @@ -1,5 +1,6 @@ === target default + >>> [CLI] bundle validate -t default -o json { "sync_mode": null, @@ -9,6 +10,7 @@ } === target overwrite + >>> [CLI] bundle validate -t overwrite -o json { "sync_mode": "overwrite", @@ -18,6 +20,7 @@ } === target in-place + >>> [CLI] bundle validate -t in-place -o json Warning: source-linked deployment is available only in the Databricks Workspace at workspace.sync_mode @@ -31,6 +34,7 @@ Warning: source-linked deployment is available only in the Databricks Workspace } === target immutable + >>> [CLI] bundle validate -t immutable -o json { "sync_mode": "immutable", @@ -40,6 +44,7 @@ Warning: source-linked deployment is available only in the Databricks Workspace } === target source_linked_enabled + >>> [CLI] bundle validate -t source_linked_enabled -o json Warning: presets.source_linked_deployment is deprecated, please set workspace.sync_mode to "in-place" instead at presets.source_linked_deployment @@ -57,6 +62,7 @@ Warning: source-linked deployment is available only in the Databricks Workspace } === target source_linked_disabled + >>> [CLI] bundle validate -t source_linked_disabled -o json Warning: presets.source_linked_deployment is deprecated, please set workspace.sync_mode to "overwrite" instead at presets.source_linked_deployment @@ -70,6 +76,7 @@ Warning: presets.source_linked_deployment is deprecated, please set workspace.sy } === target source_linked_with_sync_mode + >>> [CLI] bundle validate -t source_linked_with_sync_mode -o json Warning: presets.source_linked_deployment is deprecated and ignored because workspace.sync_mode is set, please remove it at presets.source_linked_deployment @@ -83,6 +90,7 @@ Warning: presets.source_linked_deployment is deprecated and ignored because work } === target invalid + >>> [CLI] bundle validate -t invalid Error: invalid value "inplace" for workspace.sync_mode (expected one of [overwrite in-place immutable]) at workspace.sync_mode @@ -96,6 +104,7 @@ Workspace: Found 1 error === target variable + >>> [CLI] bundle validate -t variable Error: invalid value "${var.mode}" for workspace.sync_mode (expected one of [overwrite in-place immutable]) at workspace.sync_mode @@ -109,6 +118,7 @@ Workspace: Found 1 error === target immutable_file_path + >>> [CLI] bundle validate -t immutable_file_path Error: workspace.file_path cannot be configured when workspace.sync_mode is immutable in databricks.yml:50:18 diff --git a/acceptance/bundle/validate/sync_mode_immutable_folder/output.txt b/acceptance/bundle/validate/sync_mode_immutable_folder/output.txt index 87f7b2c1578..a88f6fe3ff7 100644 --- a/acceptance/bundle/validate/sync_mode_immutable_folder/output.txt +++ b/acceptance/bundle/validate/sync_mode_immutable_folder/output.txt @@ -1,5 +1,6 @@ === target default + >>> [CLI] bundle validate -t default -o json Warning: experimental.immutable_folder is deprecated, please set workspace.sync_mode to "immutable" instead at experimental.immutable_folder @@ -13,6 +14,7 @@ Warning: experimental.immutable_folder is deprecated, please set workspace.sync_ } === target source_linked_disabled + >>> [CLI] bundle validate -t source_linked_disabled -o json Warning: experimental.immutable_folder is deprecated, please set workspace.sync_mode to "immutable" instead at experimental.immutable_folder @@ -30,6 +32,7 @@ Warning: presets.source_linked_deployment is deprecated and ignored because expe } === target source_linked_enabled + >>> [CLI] bundle validate -t source_linked_enabled -o json Warning: experimental.immutable_folder is deprecated, please set workspace.sync_mode to "immutable" instead at experimental.immutable_folder @@ -47,6 +50,7 @@ Warning: presets.source_linked_deployment is deprecated and ignored because expe } === target with_sync_mode + >>> [CLI] bundle validate -t with_sync_mode -o json Warning: experimental.immutable_folder is deprecated and ignored because workspace.sync_mode is set, please remove it at experimental.immutable_folder diff --git a/acceptance/bundle/variables/file-defaults/output.txt b/acceptance/bundle/variables/file-defaults/output.txt index a57a387e4e8..771f8d37bd6 100644 --- a/acceptance/bundle/variables/file-defaults/output.txt +++ b/acceptance/bundle/variables/file-defaults/output.txt @@ -1,5 +1,6 @@ === variable file + >>> [CLI] bundle validate -o json { "job_cluster_key": "mlops_stacks-cluster", @@ -10,6 +11,7 @@ } === variable file and variable flag + >>> [CLI] bundle validate -o json --var=cluster_key=mlops_stacks-cluster-overriden { "job_cluster_key": "mlops_stacks-cluster-overriden", @@ -20,6 +22,7 @@ } === variable file and environment variable + >>> BUNDLE_VAR_cluster_key=mlops_stacks-cluster-overriden [CLI] bundle validate -o json { "job_cluster_key": "mlops_stacks-cluster-overriden", @@ -30,6 +33,7 @@ } === variable has value in config file + >>> [CLI] bundle validate -o json --target with_value { "job_cluster_key": "mlops_stacks-cluster-from-file", @@ -40,6 +44,7 @@ } === file cannot be parsed + >>> errcode [CLI] bundle validate -o json --target invalid_json Error: failed to parse variables file [TEST_TMP_DIR]/.databricks/bundle/invalid_json/variable-overrides.json: error decoding JSON at [TEST_TMP_DIR]/.databricks/bundle/invalid_json/variable-overrides.json:1:1: invalid character 'o' in literal false (expecting 'a') @@ -54,6 +59,7 @@ Exit code: 1 } === file has wrong structure + >>> errcode [CLI] bundle validate -o json --target wrong_file_structure Error: failed to parse variables file [TEST_TMP_DIR]/.databricks/bundle/wrong_file_structure/variable-overrides.json: invalid format @@ -71,6 +77,7 @@ Exit code: 1 } === file has variable that is complex but default is string + >>> errcode [CLI] bundle validate -o json --target complex_to_string Error: variable cluster_key is not of type complex, but the value in the variable file is a complex type @@ -85,6 +92,7 @@ Exit code: 1 } === file has variable that is string but default is complex + >>> errcode [CLI] bundle validate -o json --target string_to_complex Error: variable cluster is of type complex, but the value in the variable file is not a complex type @@ -99,6 +107,7 @@ Exit code: 1 } === variable is required but it's not provided in the file + >>> errcode [CLI] bundle validate -o json --target without_defaults Error: no value assigned to required variable cluster. Variables are usually assigned in databricks.yml, and they can be overridden using "--var", the BUNDLE_VAR_cluster environment variable, or .databricks/bundle//variable-overrides.json diff --git a/acceptance/bundle/variables/locations/output.txt b/acceptance/bundle/variables/locations/output.txt index aa7897fbaba..c95e7b15912 100644 --- a/acceptance/bundle/variables/locations/output.txt +++ b/acceptance/bundle/variables/locations/output.txt @@ -1,5 +1,6 @@ === Resolved variable values + >>> [CLI] bundle summary -o json --include-locations --var from_cli=cli-value { "complex_var": { @@ -19,6 +20,7 @@ } === Variable locations as file:line:column + >>> [CLI] bundle summary -o json --include-locations --var from_cli=cli-value { "variables": [ diff --git a/acceptance/bundle/variables/reference-typo/script b/acceptance/bundle/variables/reference-typo/script index 9f6d7b3fd2d..97ed2e78719 100644 --- a/acceptance/bundle/variables/reference-typo/script +++ b/acceptance/bundle/variables/reference-typo/script @@ -1,20 +1,20 @@ -title "Typo of a defined variable: single suggestion\n" +title "Typo of a defined variable: single suggestion" withdir single musterr trace $CLI bundle validate -title "Typo close to two defined variables: multiple suggestions\n" +title "Typo close to two defined variables: multiple suggestions" withdir multiple musterr trace $CLI bundle validate -title "Typo in a non-variable reference (workspace field): multiple suggestions\n" +title "Typo in a non-variable reference (workspace field): multiple suggestions" withdir non-var-multiple musterr trace $CLI bundle validate -title "Nested complex variable with multiple typos: only the first is reported\n" +title "Nested complex variable with multiple typos: only the first is reported" withdir multiple-typos musterr trace $CLI bundle validate -title "Two typos in one reference: lookup fails at the outermost key\n" +title "Two typos in one reference: lookup fails at the outermost key" withdir double-nested-typo musterr trace $CLI bundle validate -title "Two typos in one reference, outermost close to two variables: multiple suggestions\n" +title "Two typos in one reference, outermost close to two variables: multiple suggestions" withdir double-nested-typo-multiple musterr trace $CLI bundle validate -title "Volume schema and catalog references missing resources\n" +title "Volume schema and catalog references missing resources" withdir volume-missing-resources musterr trace $CLI bundle validate diff --git a/acceptance/bundle/variables/sensitive/script b/acceptance/bundle/variables/sensitive/script index 04a225a8d00..9ad47938228 100644 --- a/acceptance/bundle/variables/sensitive/script +++ b/acceptance/bundle/variables/sensitive/script @@ -1,11 +1,11 @@ -title "A variable used as the value of a secret is sensitive and redacted in validate -o json\n" +title "A variable used as the value of a secret is sensitive and redacted in validate -o json" withdir redact trace $CLI bundle validate -o json | contains.py '!dummy-credential' | jq '{variables, resources}' -title "Sensitive variable used in non-sensitive resource fields\n" +title "Sensitive variable used in non-sensitive resource fields" withdir invalid-resource-field musterr trace $CLI bundle validate -title "Sensitive variable used in the default of a non-sensitive variable\n" +title "Sensitive variable used in the default of a non-sensitive variable" withdir invalid-variable musterr trace $CLI bundle validate -title "Sensitive variable passed as the value of a non-sensitive variable\n" +title "Sensitive variable passed as the value of a non-sensitive variable" withdir redact musterr trace $CLI bundle validate --var 'target_schema=${var.api_credential}' diff --git a/acceptance/cache/clear/script b/acceptance/cache/clear/script index 7c082ae82dd..df4edc294fc 100644 --- a/acceptance/cache/clear/script +++ b/acceptance/cache/clear/script @@ -2,15 +2,15 @@ export DATABRICKS_CACHE_DIR=$(pwd)/.cache $CLI cache clear &> /dev/null -title "First call in a session is expected to be a cache miss:\n" +title "First call in a session is expected to be a cache miss:" trace $CLI bundle validate --debug 2>&1 | grep "Local Cache" | grep -v "cache path" -title "Second call in a session is expected to be a cache hit\n" +title "Second call in a session is expected to be a cache hit" trace $CLI bundle validate --debug 2>&1 | grep "Local Cache" | grep -v "cache path" trace $CLI cache clear -title "First call after a clear is expected to be a cache miss:\n" +title "First call after a clear is expected to be a cache miss:" trace $CLI bundle validate --debug 2>&1 | grep "Local Cache" | grep -v "cache path" rm -rf "${DATABRICKS_CACHE_DIR}" diff --git a/acceptance/cache/simple/script b/acceptance/cache/simple/script index 700183f096d..c0d69d43f78 100644 --- a/acceptance/cache/simple/script +++ b/acceptance/cache/simple/script @@ -2,13 +2,13 @@ export DATABRICKS_CACHE_DIR=$(pwd)/.cache $CLI cache clear &> /dev/null -title "First call in a session is expected to be a cache miss:\n" +title "First call in a session is expected to be a cache miss:" trace $CLI bundle validate -p dogfood --debug 2>&1 | grep "Local Cache" | grep -v "cache path" -title "Second call in a session is expected to be a cache hit\n" +title "Second call in a session is expected to be a cache hit" trace $CLI bundle validate -p dogfood --debug 2>&1 | grep "Local Cache" | grep -v "cache path" -title "Bundle deploy should send telemetry values\n" +title "Bundle deploy should send telemetry values" trace $CLI bundle deploy -p dogfood trace print_telemetry_bool_values | grep "local.cache" diff --git a/acceptance/cmd/aitools/claude-config-dir/script b/acceptance/cmd/aitools/claude-config-dir/script index bdec19ef438..26f51369813 100644 --- a/acceptance/cmd/aitools/claude-config-dir/script +++ b/acceptance/cmd/aitools/claude-config-dir/script @@ -3,16 +3,16 @@ unset CLAUDE_CODE_SESSION_ID # aitools commands skip the install hint; version exercises it without auth or network. sethome ./empty-home -title "Plugin installed only in the custom config directory\n" +title "Plugin installed only in the custom config directory" trace CLAUDE_CONFIG_DIR=custom $CLI version sethome ./installed-home -title "Custom directory takes precedence over the default installation\n" +title "Custom directory takes precedence over the default installation" trace CLAUDE_CONFIG_DIR=missing $CLI version -title "Empty override uses the default installation\n" +title "Empty override uses the default installation" trace CLAUDE_CONFIG_DIR= $CLI version -title "Unset override uses the default installation\n" +title "Unset override uses the default installation" unset CLAUDE_CONFIG_DIR trace $CLI version diff --git a/acceptance/cmd/aitools/list/explicit-ref/script b/acceptance/cmd/aitools/list/explicit-ref/script index 88653209b47..699d0c0c500 100644 --- a/acceptance/cmd/aitools/list/explicit-ref/script +++ b/acceptance/cmd/aitools/list/explicit-ref/script @@ -3,5 +3,5 @@ sethome home trace $CLI aitools list --output json | list_fields -title "Only the skills manifest reaches the mock server: no latest-release lookup\n" +title "Only the skills manifest reaches the mock server: no latest-release lookup" trace print_requests.py --get diff --git a/acceptance/cmd/aitools/list/latest-lookup-fails/script b/acceptance/cmd/aitools/list/latest-lookup-fails/script index 7d44f3d74c6..b37d6deb411 100644 --- a/acceptance/cmd/aitools/list/latest-lookup-fails/script +++ b/acceptance/cmd/aitools/list/latest-lookup-fails/script @@ -6,5 +6,5 @@ use_compat_manifest # failed latest-release lookup fails the command. trace musterr $CLI aitools list --output json -title "The lookup fails, so the manifest is never fetched\n" +title "The lookup fails, so the manifest is never fetched" trace print_requests.py --get diff --git a/acceptance/cmd/aitools/list/latest/script b/acceptance/cmd/aitools/list/latest/script index e7a3f89da84..aa052eb6643 100644 --- a/acceptance/cmd/aitools/list/latest/script +++ b/acceptance/cmd/aitools/list/latest/script @@ -4,5 +4,5 @@ use_compat_manifest trace $CLI aitools list --output json | list_fields -title "The latest-release lookup picks the manifest version\n" +title "The latest-release lookup picks the manifest version" trace print_requests.py --get diff --git a/acceptance/cmd/aitools/list/pinned/script b/acceptance/cmd/aitools/list/pinned/script index 2125ca20877..0f5c2fe1bb7 100644 --- a/acceptance/cmd/aitools/list/pinned/script +++ b/acceptance/cmd/aitools/list/pinned/script @@ -4,5 +4,5 @@ use_compat_manifest trace $CLI aitools list --output json | list_fields -title "A pinned version skips the latest-release lookup\n" +title "A pinned version skips the latest-release lookup" trace print_requests.py --get diff --git a/acceptance/cmd/api/default-profile-vs-env/script b/acceptance/cmd/api/default-profile-vs-env/script index b8f620081ac..a6b4c91b398 100644 --- a/acceptance/cmd/api/default-profile-vs-env/script +++ b/acceptance/cmd/api/default-profile-vs-env/script @@ -4,6 +4,6 @@ sethome "./home" # (basic) auth on a different host. The env below still points a PAT at the test # server. Without the #5616 guard, the default profile would be pinned and merged # with the env PAT, failing with "more than one authorization method configured". -title "api without --profile uses env auth, ignoring default_profile (#5616)\n" +title "api without --profile uses env auth, ignoring default_profile (#5616)" MSYS_NO_PATHCONV=1 trace $CLI api get /api/2.0/clusters/list trace print_requests.py --get //api/2.0/clusters/list diff --git a/acceptance/cmd/api/default-profile/script b/acceptance/cmd/api/default-profile/script index 8f0ea946e66..0a83edc8ffa 100644 --- a/acceptance/cmd/api/default-profile/script +++ b/acceptance/cmd/api/default-profile/script @@ -9,9 +9,9 @@ unset DATABRICKS_HOST unset DATABRICKS_TOKEN unset DATABRICKS_CONFIG_PROFILE -title "default_profile is used when no --profile flag and no DATABRICKS_CONFIG_PROFILE\n" +title "default_profile is used when no --profile flag and no DATABRICKS_CONFIG_PROFILE" MSYS_NO_PATHCONV=1 trace $CLI api get /api/2.0/clusters/list trace print_requests.py --get //api/2.0/clusters/list | contains.py "X-Databricks-Workspace-Id" -title "--profile overrides default_profile (negative case)\n" +title "--profile overrides default_profile (negative case)" MSYS_NO_PATHCONV=1 errcode $CLI api get /api/2.0/clusters/list -p other 2>&1 | contains.py "other.test" diff --git a/acceptance/cmd/api/integer-precision/output.txt b/acceptance/cmd/api/integer-precision/output.txt index 02c843db4b4..a0e80092f0b 100644 --- a/acceptance/cmd/api/integer-precision/output.txt +++ b/acceptance/cmd/api/integer-precision/output.txt @@ -1,5 +1,6 @@ === Response: a >2^53 id is rendered exactly, in server key order + >>> [CLI] api get /api/2.0/test/bigint { "name": "a>> print_requests.py //test/bigint --del-field headers { "method": "POST", @@ -18,6 +20,7 @@ } === Request query (GET): the id in --json survives in the query string + >>> print_requests.py --get //test/bigint --del-field headers { "method": "GET", diff --git a/acceptance/cmd/api/profile-overrides-env/script b/acceptance/cmd/api/profile-overrides-env/script index 3628f59e9c3..5bab9860d38 100644 --- a/acceptance/cmd/api/profile-overrides-env/script +++ b/acceptance/cmd/api/profile-overrides-env/script @@ -10,13 +10,13 @@ real_token=$DATABRICKS_TOKEN export DATABRICKS_HOST=https://dev.cloud.databricks.test export DATABRICKS_TOKEN=dev-token -title "api --profile overrides auth env vars (#5096)\n" +title "api --profile overrides auth env vars (#5096)" MSYS_NO_PATHCONV=1 trace $CLI api get /api/2.0/clusters/list --profile my-workspace trace print_requests.py --get //api/2.0/clusters/list # Host-only profile: the profile wins for the host, but env fills the token it # omits (#5096). export DATABRICKS_TOKEN=$real_token -title "api host-only --profile fills the token from the environment (#5096)\n" +title "api host-only --profile fills the token from the environment (#5096)" MSYS_NO_PATHCONV=1 trace $CLI api get /api/2.0/clusters/list --profile host-only trace print_requests.py --get //api/2.0/clusters/list diff --git a/acceptance/cmd/api/request-handling/output.txt b/acceptance/cmd/api/request-handling/output.txt index cb4438fa337..467db6038ec 100644 --- a/acceptance/cmd/api/request-handling/output.txt +++ b/acceptance/cmd/api/request-handling/output.txt @@ -1,8 +1,10 @@ === Accept: application/json is sent (the raw response target must not drop it) + OK: Accept header present === A literal 'null' --json sends no request body (not the bytes "null") + >>> print_requests.py //test/req --del-field headers { "method": "POST", @@ -10,6 +12,7 @@ OK: Accept header present } === Query parameters in the path and a --json body are both sent + >>> print_requests.py //test/req --del-field headers { "method": "POST", diff --git a/acceptance/cmd/api/request-handling/script b/acceptance/cmd/api/request-handling/script index c85bccd5df9..f9efdb0396c 100644 --- a/acceptance/cmd/api/request-handling/script +++ b/acceptance/cmd/api/request-handling/script @@ -13,5 +13,5 @@ title "Query parameters in the path and a --json body are both sent" MSYS_NO_PATHCONV=1 $CLI api post "/api/2.0/test/req?foo=bar&x=1" --json '{"hello": "world"}' > /dev/null trace print_requests.py //test/req --del-field headers -title "Malformed --json is rejected client-side with a positioned error\n" +title "Malformed --json is rejected client-side with a positioned error" MSYS_NO_PATHCONV=1 musterr $CLI api post /api/2.0/test/req --json '{"a": 1,}' diff --git a/acceptance/cmd/api/response-handling/output.txt b/acceptance/cmd/api/response-handling/output.txt index f1724aef023..11719fed44b 100644 --- a/acceptance/cmd/api/response-handling/output.txt +++ b/acceptance/cmd/api/response-handling/output.txt @@ -1,16 +1,20 @@ === Empty response body renders null + >>> [CLI] api get /api/2.0/test/empty null === A trailing newline in the body does not add a blank line + >>> [CLI] api get /api/2.0/test/trailing { "ok": true } === An HTML response surfaces the SDK's actionable error + OK: HTML detected === A malformed JSON response surfaces an unmarshal error + OK: unmarshal error reported diff --git a/acceptance/cmd/apps/init/package_manager/output.txt b/acceptance/cmd/apps/init/package_manager/output.txt index 58f81bb908a..716c07f7576 100644 --- a/acceptance/cmd/apps/init/package_manager/output.txt +++ b/acceptance/cmd/apps/init/package_manager/output.txt @@ -1,5 +1,6 @@ === Missing pnpm fails before creating files, and the same name can be retried + >>> [CLI] apps init --name retry-app --template pnpm-template Error: pnpm is unavailable; use databricks apps init --package-manager npm, install pnpm with corepack enable pnpm or npx get-pnpm, or use databricks apps init --skip-install: exec: "pnpm": executable file not found in PATH @@ -19,10 +20,12 @@ Error: pnpm is unavailable; use databricks apps init --package-manager npm, inst "pnpm@10.17.1+sha224.abc" === Missing npm also fails before creating files + >>> [CLI] apps init --name missing-npm --template npm-only-template Error: npm is unavailable; install it and retry, or use databricks apps init --skip-install to scaffold without running setup: exec: "npm": executable file not found in PATH === Explicit manager values must be nonempty and supported + >>> [CLI] apps init --name invalid-manager --package-manager= --skip-install Error: --package-manager must be npm or pnpm @@ -30,6 +33,7 @@ Error: --package-manager must be npm or pnpm Error: unknown package manager "yarn" (allowed: npm, pnpm) === Python templates reject an explicit Node.js package manager + >>> [CLI] apps init --name python-app --template python-template --package-manager npm --skip-install Error: --package-manager is only supported for Node.js templates; remove the flag for this template @@ -51,6 +55,7 @@ Error: --package-manager is only supported for Node.js templates; remove the fla === Pnpm includes development dependencies and preserves the template pin + >>> NODE_ENV=production [CLI] apps init --name pnpm-installed --template pnpm-template ✔ Installing dependencies ✔ Creating project @@ -85,6 +90,7 @@ Warn: Background install failed: exit status 1, will retry during project initia pnpm@10.17.1+sha224.abc === Failed installs remove the project so the same name can be retried + >>> CLI_TEST_INSTALL_MODE=fail [CLI] apps init --name install-failed --template pnpm-template Warn: Background install failed: exit status 1, will retry during project initialization ✔ Creating project @@ -106,21 +112,25 @@ Error: Failed to install dependencies: exit status 1 === Validation after background preparation cancels installation and removes files + >>> CLI_TEST_INSTALL_MODE=wait [CLI] apps init --name validation-failed --template pnpm-template --run invalid Error: invalid --run value: "invalid" (must be none, dev, or dev-remote) === Failed background preparation is cleaned up even without an installer channel + >>> [CLI] apps init --name preparation-failed --template broken-template Warn: Failed to prepare package.json: package.json must contain a JSON object, skipping background install Error: load manifest: parse manifest: json: cannot unmarshal array into Go value of type manifest.Manifest === Failed setup removes the project + >>> CLI_TEST_NPM_VERSION=11.4.1 [CLI] apps init --name setup-failed --template setup-template ✔ Installing dependencies ✔ Creating project Error: Failed to run appkit setup: exit status 1 === In-place failures preserve the current directory and existing git metadata + >>> CLI_TEST_INSTALL_MODE=fail [CLI] apps init --name . --template ../pnpm-template Warn: Background install failed: exit status 1, will retry during project initialization ✔ Creating project @@ -128,6 +138,7 @@ Warn: scaffold failed in current directory; review and clean up generated files Error: Failed to install dependencies: exit status 1 === A failed dev command preserves the completed project + >>> [CLI] apps init --name dev-failed --template pnpm-template --run dev ✔ Installing dependencies ✔ Creating project @@ -225,6 +236,7 @@ null } === Standard template with pnpm + >>> [CLI] apps init --name standard-app-pnpm --template standard-template --skip-install ✔ Creating project @@ -260,6 +272,7 @@ command: } === Standard template with npm + >>> [CLI] apps init --name standard-app-npm --template standard-template --package-manager npm --skip-install ✔ Creating project @@ -295,6 +308,7 @@ null } === Lockless templates reject installation before creating the project + >>> [CLI] apps init --name lockless-app-npm --template standard-template --package-manager npm Error: template has no npm-shrinkwrap.json or package-lock.json required by "npm ci"; add npm-shrinkwrap.json or package-lock.json to the template or use --skip-install to scaffold without installing dependencies @@ -302,10 +316,12 @@ Error: template has no npm-shrinkwrap.json or package-lock.json required by "npm Error: template has no pnpm-lock.yaml required by "pnpm install --frozen-lockfile"; add pnpm-lock.yaml to the template or use --skip-install to scaffold without installing dependencies === Npm-only custom template rejects pnpm before copying + >>> [CLI] apps init --name incompatible-app --template npm-only-template --package-manager pnpm --skip-install Error: template has package-lock.json but no pnpm-lock.yaml required by "pnpm install --frozen-lockfile"; use --package-manager npm or add pnpm-lock.yaml to the template === Npm-only custom template selects npm quietly + >>> [CLI] apps init --name compatible-app --template npm-only-template --skip-install ✔ Creating project @@ -322,6 +338,7 @@ Error: template has package-lock.json but no pnpm-lock.yaml required by "pnpm in "npm@9.9.4" === Npm records the installed version in a plain template + >>> CLI_TEST_NPM_VERSION=11.4.1 [CLI] apps init --name installed-app --template npm-only-template --package-manager npm ✔ Installing dependencies ✔ Creating project @@ -339,6 +356,7 @@ Error: template has package-lock.json but no pnpm-lock.yaml required by "pnpm in npm@11.4.1 === Npm records the installed version in a Go template + >>> CLI_TEST_NPM_VERSION=10.8.3 [CLI] apps init --name rendered-app --template install-template --package-manager npm ✔ Installing dependencies ✔ Creating project @@ -356,18 +374,22 @@ npm@11.4.1 npm@10.8.3 === Npm version detection failure stops before creating the project + >>> [CLI] apps init --name failed-app --template npm-only-template --package-manager npm Error: detect npm version; ensure npm is installed and working, or use --skip-install: exit status 1 === Npm invalid version stops before creating the project + >>> CLI_TEST_NPM_VERSION=invalid [CLI] apps init --name invalid-app --template npm-only-template --package-manager npm Error: npm --version returned invalid version "invalid"; ensure npm is working, or use --skip-install === Npm empty version stops before creating the project + >>> CLI_TEST_NPM_VERSION= [CLI] apps init --name empty-app --template npm-only-template --package-manager npm Error: npm --version returned invalid version ""; ensure npm is working, or use --skip-install === Version-like branch keeps pnpm + >>> [CLI] apps init --name branch-app --branch 0.81-maintenance --skip-install ✔ Cloning template (branch 0.81-maintenance) ✔ Creating project @@ -391,6 +413,7 @@ command: "pnpm@11.0.8" === First dual-manager template version keeps pnpm + >>> [CLI] apps init --name dual-manager-app --version 0.82.0 --package-manager pnpm --skip-install ✔ Cloning template (version 0.82.0) ✔ Creating project @@ -415,6 +438,7 @@ command: "pnpm@11.0.8" === Version downgrade uses npm throughout + >>> [CLI] apps init --name older-app --version 0.81.0 --skip-install ✔ Cloning template (version 0.81.0) ✔ Creating project @@ -439,6 +463,7 @@ command: null === Explicit pnpm on an older template warns before selecting npm + >>> [CLI] apps init --name explicit-older-app --version 0.81.0 --package-manager pnpm --skip-install ✔ Cloning template (version 0.81.0) Warn: Package manager "pnpm" is not supported for AppKit version 0.81.0, using npm instead @@ -457,6 +482,7 @@ Warn: Package manager "pnpm" is not supported for AppKit version 0.81.0, using n null === Missing version falls back to the embedded template and its package manager + >>> DATABRICKS_CACHE_DIR=cache DATABRICKS_CACHE_ENABLED=true DATABRICKS_FORCE_EMBEDDED_COMPAT=false [CLI] apps init --name fallback-app --skip-install Warn: Template version not found, falling back to embedded version 0.84.0 ✔ Cloning template (version 0.84.0) @@ -482,6 +508,7 @@ command: "pnpm@11.0.8" === Version downgrade resolves the installed npm pin + >>> CLI_TEST_NPM_VERSION=11.4.1 [CLI] apps init --name installed-older-app --version 0.81.0 ✔ Cloning template (version 0.81.0) ✔ Installing dependencies diff --git a/acceptance/cmd/apps/init/package_manager_prune/output.txt b/acceptance/cmd/apps/init/package_manager_prune/output.txt index f32d0619553..19e56864536 100644 --- a/acceptance/cmd/apps/init/package_manager_prune/output.txt +++ b/acceptance/cmd/apps/init/package_manager_prune/output.txt @@ -34,6 +34,7 @@ npm-shrinkwrap.json package-lock.json === Npm accepts a shrinkwrap template with a pnpm lockfile + >>> [CLI] apps init --name shrinkwrap-app --template shrinkwrap-template --package-manager npm --skip-install ✔ Creating project diff --git a/acceptance/cmd/auth/describe/account-host-with-workspace-id/script b/acceptance/cmd/auth/describe/account-host-with-workspace-id/script index e2ba8c9ff19..de29ec18197 100644 --- a/acceptance/cmd/auth/describe/account-host-with-workspace-id/script +++ b/acceptance/cmd/auth/describe/account-host-with-workspace-id/script @@ -5,5 +5,5 @@ sethome "./home" # serves account APIs (https://github.com/databricks/cli/issues/5479). envsubst < "./home/.databrickscfg.tmpl" > "./home/.databrickscfg" -title "Describe falls back to the account endpoint when the workspace check fails\n" +title "Describe falls back to the account endpoint when the workspace check fails" trace $CLI auth describe --profile acct-with-ws diff --git a/acceptance/cmd/auth/describe/bundle-profile-env/script b/acceptance/cmd/auth/describe/bundle-profile-env/script index 33dc4391829..7a5677ebee1 100644 --- a/acceptance/cmd/auth/describe/bundle-profile-env/script +++ b/acceptance/cmd/auth/describe/bundle-profile-env/script @@ -11,5 +11,5 @@ export DATABRICKS_CONFIG_PROFILE=env-profile # Inside a bundle root the resolved config labels every attribute "bundle", so # before #2303 the profile read "(from bundle)" even though it came from # DATABRICKS_CONFIG_PROFILE. describe must attribute it to the env var. -title "Describe inside a bundle attributes the profile to DATABRICKS_CONFIG_PROFILE (#2303)\n" +title "Describe inside a bundle attributes the profile to DATABRICKS_CONFIG_PROFILE (#2303)" trace $CLI auth describe diff --git a/acceptance/cmd/auth/describe/default-profile/script b/acceptance/cmd/auth/describe/default-profile/script index 8a677abec6f..5faa33cbdaf 100644 --- a/acceptance/cmd/auth/describe/default-profile/script +++ b/acceptance/cmd/auth/describe/default-profile/script @@ -5,10 +5,10 @@ envsubst < "./home/.databrickscfg.tmpl" > "./home/.databrickscfg" # DATABRICKS_HOST is set in the environment, so authentication is fully # determined by the environment and the default profile is not pinned. -title "Describe without --profile (DATABRICKS_HOST takes precedence over default_profile)\n" +title "Describe without --profile (DATABRICKS_HOST takes precedence over default_profile)" trace $CLI auth describe # Without a host in the environment, the default profile is resolved and used. unset DATABRICKS_HOST DATABRICKS_TOKEN -title "Describe without --profile and without DATABRICKS_HOST (uses default_profile)\n" +title "Describe without --profile and without DATABRICKS_HOST (uses default_profile)" trace $CLI auth describe diff --git a/acceptance/cmd/auth/describe/profile-overrides-env/script b/acceptance/cmd/auth/describe/profile-overrides-env/script index cccc6917845..9837009340c 100644 --- a/acceptance/cmd/auth/describe/profile-overrides-env/script +++ b/acceptance/cmd/auth/describe/profile-overrides-env/script @@ -9,11 +9,11 @@ real_token=$DATABRICKS_TOKEN export DATABRICKS_HOST=https://dev.cloud.databricks.test export DATABRICKS_TOKEN=dev-token -title "Describe with --profile overrides auth env vars (#5096)\n" +title "Describe with --profile overrides auth env vars (#5096)" trace $CLI auth describe --profile my-workspace # Host-only profile: the profile wins for the host, but env fills the token it # omits (#5096). export DATABRICKS_TOKEN=$real_token -title "Describe with a host-only --profile fills the token from the environment (#5096)\n" +title "Describe with a host-only --profile fills the token from the environment (#5096)" trace $CLI auth describe --profile host-only diff --git a/acceptance/cmd/auth/docker/configure-errors/script b/acceptance/cmd/auth/docker/configure-errors/script index b05752e3027..da8890f5abe 100644 --- a/acceptance/cmd/auth/docker/configure-errors/script +++ b/acceptance/cmd/auth/docker/configure-errors/script @@ -33,31 +33,31 @@ workspace_id = 333444555 ENDCFG cp "$HOME/.databrickscfg" "$HOME/databrickscfg.before" -title "Unsupported inherited flags\n" +title "Unsupported inherited flags" trace musterr $CLI auth docker configure docker-test --host https://other.cloud.databricks.test trace musterr $CLI auth docker configure docker-test --account-id 00000000-0000-0000-0000-000000000000 trace musterr $CLI auth docker configure docker-test --workspace-id 900800700600 -title "Profile argument and flag\n" +title "Profile argument and flag" trace musterr $CLI auth docker configure docker-test --profile docker-test -title "Invalid explicit region\n" +title "Invalid explicit region" trace musterr $CLI auth docker configure docker-test --region= trace configure_with_whitespace_region trace musterr $CLI auth docker configure docker-test --region=invalid_region -title "Unsupported workspace host\n" +title "Unsupported workspace host" trace musterr $CLI auth docker configure unsupported-host -title "Unsupported authentication profiles\n" +title "Unsupported authentication profiles" trace musterr $CLI auth docker configure pat trace musterr $CLI auth docker configure m2m trace musterr $CLI auth docker configure blank-auth -title "Another profile already maps to the workspace\n" +title "Another profile already maps to the workspace" trace musterr $CLI auth docker configure docker-test -title "Nothing changed\n" +title "Nothing changed" assert_not_exists.py "$DOCKER_CONFIG/config.json" cmp "$HOME/databrickscfg.before" "$HOME/.databrickscfg" printf 'Docker config not created; profiles unchanged\n' diff --git a/acceptance/cmd/auth/docker/configure-missing-region/script b/acceptance/cmd/auth/docker/configure-missing-region/script index 0925a13ff8f..4b86dcad6fd 100644 --- a/acceptance/cmd/auth/docker/configure-missing-region/script +++ b/acceptance/cmd/auth/docker/configure-missing-region/script @@ -6,7 +6,7 @@ cp "$HOME/.databrickscfg" "$HOME/databrickscfg.before" trace musterr $CLI auth docker configure docker-test -title "Profile and Docker state unchanged\n" +title "Profile and Docker state unchanged" cmp "$HOME/databrickscfg.before" "$HOME/.databrickscfg" assert_not_exists.py "$DOCKER_CONFIG/config.json" assert_not_exists.py "$HOME/bin/docker-credential-databricks" diff --git a/acceptance/cmd/auth/docker/configure/script b/acceptance/cmd/auth/docker/configure/script index 96b62b88d8e..8df314ca720 100644 --- a/acceptance/cmd/auth/docker/configure/script +++ b/acceptance/cmd/auth/docker/configure/script @@ -22,10 +22,10 @@ if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" || "$OSTYPE" == "win32" ]]; t } fi -title "Docker config\n" +title "Docker config" jq -S . "$DOCKER_CONFIG/config.json" -title "Profile with resolved workspace ID\n" +title "Profile with resolved workspace ID" cat "$HOME/.databrickscfg" trace databricks auth docker host --profile docker-test --output json | tee "$HOME/host.json" @@ -34,24 +34,24 @@ trace print_requests.py --get //api registry_host=$(jq -r .host "$HOME/host.json") printf '%s\n' "$registry_host" | trace docker-credential-databricks get | jq -S . -title "Invalid registry host\n" +title "Invalid registry host" printf '%s\n' registry.example.test | trace musterr docker-credential-databricks get trace musterr docker-credential-databricks erase -title "Configure again\n" +title "Configure again" cp "$DOCKER_CONFIG/config.json" "$HOME/config.before.json" databricks auth docker configure docker-test &>LOG.configure-again cmp "$HOME/config.before.json" "$DOCKER_CONFIG/config.json" printf 'Docker config unchanged\n' trace print_requests.py --get //api -title "Deprecated --region flag skips workspace lookups\n" +title "Deprecated --region flag skips workspace lookups" trace databricks auth docker configure docker-test --region us-west-2 assert_not_exists.py "$OUT_REQUESTS" printf 'No requests sent\n' -title "Explicit --region resolves a missing workspace ID\n" +title "Explicit --region resolves a missing workspace ID" cat > "$HOME/.databrickscfg" <> "$HOME/.databrickscfg" <<'ENDCFG' @@ -17,31 +17,31 @@ auth_type = databricks-cli ENDCFG cp "$HOME/.databrickscfg" "$HOME/databrickscfg.before" -title "Workspace profile required\n" +title "Workspace profile required" trace musterr $CLI auth docker host --profile account -title "Unsupported workspace host\n" +title "Unsupported workspace host" trace musterr $CLI auth docker host --profile unsupported-host -title "Other authentication flags not supported\n" +title "Other authentication flags not supported" trace musterr $CLI auth docker host --profile docker-test --host https://other.cloud.databricks.test trace musterr $CLI auth docker host --profile docker-test --account-id 00000000-0000-0000-0000-000000000000 trace musterr $CLI auth docker host --profile docker-test --workspace-id 111222333 -title "Credential helper not configured\n" +title "Credential helper not configured" trace $CLI auth docker host --profile docker-test assert_not_exists.py "$DOCKER_CONFIG/config.json" -title "Invalid Docker config\n" +title "Invalid Docker config" mkdir -p "$DOCKER_CONFIG" cp invalid-docker-config.json "$DOCKER_CONFIG/config.json" trace musterr $CLI auth docker host --profile docker-test -title "Default credential store without an installed helper\n" +title "Default credential store without an installed helper" cp docker-config.json "$DOCKER_CONFIG/config.json" trace PATH="$HOME" $CLI auth docker host --profile docker-test trace PATH="$HOME" $CLI auth docker host --profile docker-test --output json -title "Profiles unchanged\n" +title "Profiles unchanged" cmp "$HOME/databrickscfg.before" "$HOME/.databrickscfg" printf 'Workspace ID not saved\n' diff --git a/acceptance/cmd/auth/docker/invalid-refresh-token/script b/acceptance/cmd/auth/docker/invalid-refresh-token/script index ddbfcf8c015..7bb96ab1438 100644 --- a/acceptance/cmd/auth/docker/invalid-refresh-token/script +++ b/acceptance/cmd/auth/docker/invalid-refresh-token/script @@ -2,7 +2,7 @@ setup_docker_auth cp token-cache.json "$HOME/.databricks/token-cache.json" export DOCKER_CONFIG="$HOME/docker" -title "Unresolved workspace ID\n" +title "Unresolved workspace ID" trace musterr $CLI auth docker host --profile docker-test trace musterr $CLI auth docker configure docker-test @@ -10,11 +10,11 @@ cat >> "$HOME/.databrickscfg" <<'ENDCFG' workspace_id = 900800700600 ENDCFG -title "Stored workspace ID\n" +title "Stored workspace ID" trace musterr $CLI auth docker host --profile docker-test trace musterr $CLI auth docker configure docker-test printf '%s\n' "900800700600.container.us-west-2.localhost" | trace musterr $CLI auth docker token -title "Docker config not created\n" +title "Docker config not created" assert_not_exists.py "$DOCKER_CONFIG/config.json" printf 'OK\n' diff --git a/acceptance/cmd/auth/docker/token/script b/acceptance/cmd/auth/docker/token/script index a272f6af120..154eab1406e 100644 --- a/acceptance/cmd/auth/docker/token/script +++ b/acceptance/cmd/auth/docker/token/script @@ -4,30 +4,30 @@ workspace_id = 900800700600 ENDCFG registry_host=900800700600.container.us-west-2.localhost -title "Cached token\n" +title "Cached token" printf '%s\n' "$registry_host" | trace $CLI auth docker token --no-force-refresh | jq -S . -title "Refreshed token by default\n" +title "Refreshed token by default" printf '%s\n' "$registry_host" | trace $CLI auth docker token | jq -S . -title "No profile for the workspace\n" +title "No profile for the workspace" printf '%s\n' "111222333.container.us-west-2.localhost" | trace musterr $CLI auth docker token -title "Not an Artifact Registry host\n" +title "Not an Artifact Registry host" printf '%s\n' "registry.example.test" | trace musterr $CLI auth docker token -title "Profile flag not supported\n" +title "Profile flag not supported" printf '%s\n' "$registry_host" | trace musterr $CLI auth docker token --profile docker-test -title "Positional argument not supported\n" +title "Positional argument not supported" printf '%s\n' "$registry_host" | trace musterr $CLI auth docker token docker-test -title "Other authentication flags not supported\n" +title "Other authentication flags not supported" printf '%s\n' "$registry_host" | trace musterr $CLI auth docker token --host https://other.cloud.databricks.test printf '%s\n' "$registry_host" | trace musterr $CLI auth docker token --account-id 00000000-0000-0000-0000-000000000000 printf '%s\n' "$registry_host" | trace musterr $CLI auth docker token --workspace-id 111222333 -title "Test-server registry never matches a remote workspace\n" +title "Test-server registry never matches a remote workspace" cat >> "$HOME/.databrickscfg" <<'ENDCFG' [remote] @@ -37,7 +37,7 @@ workspace_id = 444555666 ENDCFG printf '%s\n' "444555666.container.us-west-2.localhost" | trace musterr $CLI auth docker token -title "Multiple profiles for the workspace\n" +title "Multiple profiles for the workspace" cat >> "$HOME/.databrickscfg" <>> [CLI] auth login --host [DATABRICKS_URL] --profile DEFAULT --configure-serverless Profile DEFAULT was successfully saved diff --git a/acceptance/cmd/auth/login/configure-serverless/script b/acceptance/cmd/auth/login/configure-serverless/script index b25c2b91f12..643580ae270 100644 --- a/acceptance/cmd/auth/login/configure-serverless/script +++ b/acceptance/cmd/auth/login/configure-serverless/script @@ -3,7 +3,7 @@ sethome "./home" # Initial profile with cluster_id to be cleared out. envsubst < "./home/.databrickscfg.tmpl" > "./home/.databrickscfg" -title "Initial profile with cluster_id\n" +title "Initial profile with cluster_id" cat "./home/.databrickscfg" # Use a fake browser that performs a GET on the authorization URL @@ -13,7 +13,7 @@ export BROWSER="browser.py" title "Run auth login with --configure-serverless" trace $CLI auth login --host $DATABRICKS_HOST --profile DEFAULT --configure-serverless -title "Profile after auth login with --configure-serverless\n" +title "Profile after auth login with --configure-serverless" cat "./home/.databrickscfg" # Track the .databrickscfg file that was created to surface changes. diff --git a/acceptance/cmd/auth/login/custom-config-file/output.txt b/acceptance/cmd/auth/login/custom-config-file/output.txt index c586e020df7..4a33a9fbdc6 100644 --- a/acceptance/cmd/auth/login/custom-config-file/output.txt +++ b/acceptance/cmd/auth/login/custom-config-file/output.txt @@ -6,6 +6,7 @@ auth_type = pat token = old-token-123 === Login with conflicting host (should error) + >>> [CLI] auth login --host [DATABRICKS_URL] --profile custom-test Error: --profile "custom-test" has host "https://old-host.cloud.databricks.com", which conflicts with --host "[DATABRICKS_URL]". Use --profile only to select a profile diff --git a/acceptance/cmd/auth/login/custom-config-file/script b/acceptance/cmd/auth/login/custom-config-file/script index d253b4a3d8b..6320234b665 100644 --- a/acceptance/cmd/auth/login/custom-config-file/script +++ b/acceptance/cmd/auth/login/custom-config-file/script @@ -9,7 +9,7 @@ export DATABRICKS_CONFIG_FILE="./home/custom.databrickscfg" # home/custom.databrickscfg is a checked-in input: an existing custom config file # with a DIFFERENT host. Since --profile and --host conflict, the CLI should error. -title "Initial custom config file (with old host)\n" +title "Initial custom config file (with old host)" cat "./home/custom.databrickscfg" title "Login with conflicting host (should error)" diff --git a/acceptance/cmd/auth/login/device-code/non-interactive/concurrent/script b/acceptance/cmd/auth/login/device-code/non-interactive/concurrent/script index db75e365a89..8aa136e8529 100644 --- a/acceptance/cmd/auth/login/device-code/non-interactive/concurrent/script +++ b/acceptance/cmd/auth/login/device-code/non-interactive/concurrent/script @@ -1,7 +1,7 @@ sethome "./home" # The fake server issues the user codes WDJB0001, WDJB0002, ... in order. -title "A second login for the same profile replaces the first\n" +title "A second login for the same profile replaces the first" start_login LOG.first $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code start_login LOG.second $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code trace cache_keys @@ -9,7 +9,7 @@ trace musterr $CLI auth login --host $DATABRICKS_HOST --profile test --use-devic trace $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code --user-code WDJB-0002 --confirmation-code FTFA-QQKY trace print_requests.py --oneline //oidc/v1/token -title "Logins for different targets are independent\n" +title "Logins for different targets are independent" start_login LOG.a $CLI auth login --host $DATABRICKS_HOST --profile a --use-device-code start_login LOG.b $CLI auth login --host $DATABRICKS_HOST --profile b --use-device-code start_login LOG.host $CLI auth login --host $DATABRICKS_HOST --use-device-code diff --git a/acceptance/cmd/auth/login/device-code/non-interactive/errors/script b/acceptance/cmd/auth/login/device-code/non-interactive/errors/script index 00984e93321..3817c695ebd 100644 --- a/acceptance/cmd/auth/login/device-code/non-interactive/errors/script +++ b/acceptance/cmd/auth/login/device-code/non-interactive/errors/script @@ -1,59 +1,59 @@ sethome "./home" -title "Device code login is disabled by default\n" +title "Device code login is disabled by default" ( unset DATABRICKS_CLI_DEVICE_CODE trace musterr $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code trace musterr $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code --user-code WDJB-0001 --confirmation-code FTFA-QQKY ) -title "Flags the non-interactive login cannot honor\n" +title "Flags the non-interactive login cannot honor" trace musterr $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code --configure-cluster # The non-interactive login cannot answer the profile picker or the host prompt. -title "Missing host\n" +title "Missing host" trace musterr $CLI auth login --use-device-code trace musterr $CLI auth login --profile test --use-device-code --user-code WDJB-0001 --confirmation-code FTFA-QQKY -title "Incomplete flags of the second step\n" +title "Incomplete flags of the second step" trace musterr $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code --confirmation-code FTFA-QQKY trace musterr $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code --user-code WDJB-0001 trace musterr $CLI auth login --host $DATABRICKS_HOST --profile test --user-code WDJB-0001 --confirmation-code FTFA-QQKY -title "Profile name that marks a pending login\n" +title "Profile name that marks a pending login" trace musterr $CLI auth login --host $DATABRICKS_HOST --profile 'test#device-code' --use-device-code -title "Account host\n" +title "Account host" trace musterr $CLI auth login --host https://accounts.cloud.databricks.test --use-device-code --user-code WDJB-0001 --confirmation-code FTFA-QQKY envsubst < "./home/.databrickscfg.tmpl" > "./home/.databrickscfg" -title "Profile scoped to resources\n" +title "Profile scoped to resources" trace musterr $CLI auth login --profile resources --use-device-code --user-code WDJB-0001 --confirmation-code FTFA-QQKY -title "No pending login\n" +title "No pending login" trace musterr $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code --user-code WDJB-0001 --confirmation-code FTFA-QQKY trace musterr $CLI auth login --host $DATABRICKS_HOST --use-device-code --user-code WDJB-0001 --confirmation-code FTFA-QQKY start_login LOG.wrong-code $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code -title "Wrong confirmation code\n" +title "Wrong confirmation code" trace musterr $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code --user-code WDJB-0001 --confirmation-code AAAA-AAAA trace AGENT=test-agent musterr $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code --user-code WDJB-0001 --confirmation-code AAAA-AAAA # A wrong confirmation code leaves the device code valid, so the pending # login is kept. -title "Rerun with the right confirmation code\n" +title "Rerun with the right confirmation code" trace $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code --user-code wdjb-0001 --confirmation-code ftfa-qqky -title "Rerun after the login was completed\n" +title "Rerun after the login was completed" trace musterr $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code --user-code WDJB-0001 --confirmation-code FTFA-QQKY # The server discards the device code when the login expires. The fake server # only discards a device code once it is exchanged, so a pending login restored # from before the exchange stands in for an expired one. The pending login is # then deleted. -title "Device code unknown to the server\n" +title "Device code unknown to the server" start_login LOG.expired $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code trace cp home/.databricks/token-cache.json home/token-cache.pending.json trace $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code --user-code WDJB-0002 --confirmation-code FTFA-QQKY @@ -63,7 +63,7 @@ trace cache_keys # The token gets the scopes of the device authorization, so the second step # cannot change them. -title "Scopes changed between the steps\n" +title "Scopes changed between the steps" start_login LOG.scopes $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code --scopes sql trace musterr $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code --user-code WDJB-0003 --confirmation-code FTFA-QQKY diff --git a/acceptance/cmd/auth/login/device-code/non-interactive/flags/script b/acceptance/cmd/auth/login/device-code/non-interactive/flags/script index 2841f3c6ce8..2cc9ade2639 100644 --- a/acceptance/cmd/auth/login/device-code/non-interactive/flags/script +++ b/acceptance/cmd/auth/login/device-code/non-interactive/flags/script @@ -3,13 +3,13 @@ envsubst < "./home/.databrickscfg.tmpl" > "./home/.databrickscfg" # The rerun resolves the login from the same flags and profile as the first # step, so the token exchange uses the client ID of the profile. -title "Scopes and client ID from the profile\n" +title "Scopes and client ID from the profile" trace $CLI auth login --profile scoped --use-device-code trace $CLI auth login --profile scoped --use-device-code --user-code WDJB-0001 --confirmation-code FTFA-QQKY trace print_requests.py --oneline //oidc/v1/device/auth //oidc/v1/token # Only the second step saves the profile. -title "Serverless compute is configured in the second step\n" +title "Serverless compute is configured in the second step" trace $CLI auth login --host $DATABRICKS_HOST --profile serverless --use-device-code --configure-serverless trace $CLI auth login --host $DATABRICKS_HOST --profile serverless --use-device-code --configure-serverless --user-code WDJB-0002 --confirmation-code FTFA-QQKY diff --git a/acceptance/cmd/auth/login/device-code/non-interactive/nominal/script b/acceptance/cmd/auth/login/device-code/non-interactive/nominal/script index 635fb3533f2..c35f92c24ab 100644 --- a/acceptance/cmd/auth/login/device-code/non-interactive/nominal/script +++ b/acceptance/cmd/auth/login/device-code/non-interactive/nominal/script @@ -2,36 +2,36 @@ sethome "./home" # Acceptance tests run the CLI without a terminal on stdin, so device code # login runs non-interactively. -title "Start the login without a profile\n" +title "Start the login without a profile" trace $CLI auth login --host $DATABRICKS_HOST --use-device-code -title "The first step stores the device code and saves no profile\n" +title "The first step stores the device code and saves no profile" trace find.py home trace cache_keys -title "Confirm the login\n" +title "Confirm the login" trace $CLI auth login --host $DATABRICKS_HOST --use-device-code --user-code WDJB-0001 --confirmation-code FTFA-QQKY trace $CLI auth token --host $DATABRICKS_HOST | gron.py | grep access_token trace cache_keys -title "Start the login with a profile\n" +title "Start the login with a profile" trace $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code -title "Confirm the login\n" +title "Confirm the login" trace $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code --user-code WDJB-0002 --confirmation-code FTFA-QQKY trace $CLI auth token --profile test | gron.py | grep access_token trace $CLI auth profiles # The pending login is stored next to the token, so the token keeps working # until the login is confirmed. -title "Start the login again with the host from the saved profile\n" +title "Start the login again with the host from the saved profile" trace $CLI auth login --profile test --use-device-code trace $CLI auth token --profile test | gron.py | grep access_token trace $CLI auth login --profile test --use-device-code --user-code WDJB-0003 --confirmation-code FTFA-QQKY trace cache_keys trace print_requests.py --oneline //oidc/v1/device/auth //oidc/v1/token -title "An AI agent gets notes on passing the login on to the user\n" +title "An AI agent gets notes on passing the login on to the user" trace AGENT=test-agent $CLI auth login --host $DATABRICKS_HOST --profile agent --use-device-code trace AGENT=test-agent $CLI auth login --host $DATABRICKS_HOST --profile agent --use-device-code --user-code WDJB-0004 --confirmation-code FTFA-QQKY diff --git a/acceptance/cmd/auth/login/device-code/precondition/script b/acceptance/cmd/auth/login/device-code/precondition/script index c3a7a3208c0..7df7da78d3f 100644 --- a/acceptance/cmd/auth/login/device-code/precondition/script +++ b/acceptance/cmd/auth/login/device-code/precondition/script @@ -1,12 +1,12 @@ sethome "./home" -title "Device code login is disabled by default\n" +title "Device code login is disabled by default" ( unset DATABRICKS_CLI_DEVICE_CODE trace musterr $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code ) -title "Inputs device code login cannot honor are rejected\n" +title "Inputs device code login cannot honor are rejected" trace musterr $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code --resource https://resource.test trace musterr $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code --skip-workspace diff --git a/acceptance/cmd/auth/login/device-code/unsupported-host/script b/acceptance/cmd/auth/login/device-code/unsupported-host/script index 2d0a8949c8e..46c307b332e 100644 --- a/acceptance/cmd/auth/login/device-code/unsupported-host/script +++ b/acceptance/cmd/auth/login/device-code/unsupported-host/script @@ -1,6 +1,6 @@ sethome "./home" -title "No host\n" +title "No host" # The profile does not exist yet and no host is given. auth login does not read # DATABRICKS_HOST, so the usual login would fall back to the browser-based # discovery flow. Device code login skips it and asks for the host instead, @@ -11,18 +11,18 @@ envsubst < "./home/.databrickscfg.tmpl" > "./home/.databrickscfg" # The host is rejected before the CLI would prompt for an account ID, whether # it is passed directly or comes from a profile without one. -title "Account host\n" +title "Account host" trace musterr $CLI auth login --host https://accounts.cloud.databricks.test --profile test --use-device-code trace musterr $CLI auth login --profile account --use-device-code # An account-scoped discovery URL marks the host as a unified host. -title "Unified host\n" +title "Unified host" trace DATABRICKS_DISCOVERY_URL="$DATABRICKS_HOST/oidc/accounts/spog-acct-123/.well-known/oauth-authorization-server" musterr $CLI auth login --host $DATABRICKS_HOST --use-device-code -title "Workspace without device code login\n" +title "Workspace without device code login" trace musterr $CLI auth login --host $DATABRICKS_HOST --profile test --use-device-code trace musterr $CLI auth login --profile workspace --use-device-code # Every host is rejected before the device authorization request. -title "Device authorization requests\n" +title "Device authorization requests" trace print_requests.py //oidc/v1/device/auth diff --git a/acceptance/cmd/auth/login/host-arg-overrides-profile/output.txt b/acceptance/cmd/auth/login/host-arg-overrides-profile/output.txt index d6d8c1e3be1..a64f4534e75 100644 --- a/acceptance/cmd/auth/login/host-arg-overrides-profile/output.txt +++ b/acceptance/cmd/auth/login/host-arg-overrides-profile/output.txt @@ -4,6 +4,7 @@ host = https://old-host.cloud.databricks.com === Login with --host flag (should error on conflict) + >>> [CLI] auth login --host [DATABRICKS_URL] --profile override-test Error: --profile "override-test" has host "https://old-host.cloud.databricks.com", which conflicts with --host "[DATABRICKS_URL]". Use --profile only to select a profile diff --git a/acceptance/cmd/auth/login/host-arg-overrides-profile/script b/acceptance/cmd/auth/login/host-arg-overrides-profile/script index 44badb0d1c4..854d810f117 100644 --- a/acceptance/cmd/auth/login/host-arg-overrides-profile/script +++ b/acceptance/cmd/auth/login/host-arg-overrides-profile/script @@ -1,7 +1,7 @@ sethome "./home" # home/.databrickscfg is a checked-in input: an existing profile with a DIFFERENT host. -title "Initial profile with old host\n" +title "Initial profile with old host" cat "./home/.databrickscfg" # Login with --profile and --host where the profile has a different host. diff --git a/acceptance/cmd/auth/login/host-from-profile/output.txt b/acceptance/cmd/auth/login/host-from-profile/output.txt index 9585b8cd388..b37b3240587 100644 --- a/acceptance/cmd/auth/login/host-from-profile/output.txt +++ b/acceptance/cmd/auth/login/host-from-profile/output.txt @@ -6,6 +6,7 @@ auth_type = databricks-cli client_id = custom-client-id === Login with existing profile (no host argument) + >>> [CLI] auth login --profile existing-profile Profile existing-profile was successfully saved diff --git a/acceptance/cmd/auth/login/host-from-profile/script b/acceptance/cmd/auth/login/host-from-profile/script index c800d378fea..5b09df96d94 100644 --- a/acceptance/cmd/auth/login/host-from-profile/script +++ b/acceptance/cmd/auth/login/host-from-profile/script @@ -3,7 +3,7 @@ sethome "./home" # An existing profile with a host. envsubst < "./home/.databrickscfg.tmpl" > "./home/.databrickscfg" -title "Initial profile\n" +title "Initial profile" cat "./home/.databrickscfg" # Use a fake browser that performs a GET on the authorization URL @@ -14,16 +14,16 @@ export BROWSER="browser.py" title "Login with existing profile (no host argument)" trace $CLI auth login --profile existing-profile -title "OAuth client ID\n" +title "OAuth client ID" trace print_requests.py //oidc/v1/authorize --get | gron.py | grep client_id -title "Login with a client ID override\n" +title "Login with a client ID override" trace $CLI auth login --profile existing-profile --client-id flag-client-id -title "Overridden OAuth client ID\n" +title "Overridden OAuth client ID" trace print_requests.py //oidc/v1/authorize --get | gron.py | grep client_id -title "Profile after login\n" +title "Profile after login" cat "./home/.databrickscfg" # Track the .databrickscfg file that was created to surface changes. diff --git a/acceptance/cmd/auth/login/preserve-fields/output.txt b/acceptance/cmd/auth/login/preserve-fields/output.txt index 2e8cd590fc0..f5db1e0989e 100644 --- a/acceptance/cmd/auth/login/preserve-fields/output.txt +++ b/acceptance/cmd/auth/login/preserve-fields/output.txt @@ -8,6 +8,7 @@ azure_environment = USGOVERNMENT custom_key = my-custom-value === Run auth login (no --configure-cluster or --configure-serverless) + >>> [CLI] auth login --host [DATABRICKS_URL] --profile DEFAULT Profile DEFAULT was successfully saved diff --git a/acceptance/cmd/auth/login/preserve-fields/script b/acceptance/cmd/auth/login/preserve-fields/script index b416038a3e3..239864a17b1 100644 --- a/acceptance/cmd/auth/login/preserve-fields/script +++ b/acceptance/cmd/auth/login/preserve-fields/script @@ -4,7 +4,7 @@ sethome "./home" # key that is not a recognized SDK config attribute. envsubst < "./home/.databrickscfg.tmpl" > "./home/.databrickscfg" -title "Initial profile\n" +title "Initial profile" cat "./home/.databrickscfg" # Use a fake browser that performs a GET on the authorization URL @@ -14,5 +14,5 @@ export BROWSER="browser.py" title "Run auth login (no --configure-cluster or --configure-serverless)" trace $CLI auth login --host $DATABRICKS_HOST --profile DEFAULT -title "Profile after auth login — all non-auth fields should be preserved\n" +title "Profile after auth login — all non-auth fields should be preserved" cat "./home/.databrickscfg" diff --git a/acceptance/cmd/auth/logout/default-profile/output.txt b/acceptance/cmd/auth/logout/default-profile/output.txt index b8ea55a41c7..58db5ce07f3 100644 --- a/acceptance/cmd/auth/logout/default-profile/output.txt +++ b/acceptance/cmd/auth/logout/default-profile/output.txt @@ -11,6 +11,7 @@ host = [DATABRICKS_URL] auth_type = databricks-cli === Delete the DEFAULT profile + >>> [CLI] auth logout --profile DEFAULT --auto-approve --delete Logged out of and deleted profile "DEFAULT". diff --git a/acceptance/cmd/auth/logout/default-profile/script b/acceptance/cmd/auth/logout/default-profile/script index 367175b945f..b5b0be2042f 100644 --- a/acceptance/cmd/auth/logout/default-profile/script +++ b/acceptance/cmd/auth/logout/default-profile/script @@ -2,16 +2,16 @@ sethome "./home" envsubst < "./home/.databrickscfg.tmpl" > "./home/.databrickscfg" -title "Initial config\n" +title "Initial config" cat "./home/.databrickscfg" title "Delete the DEFAULT profile" trace $CLI auth logout --profile DEFAULT --auto-approve --delete -title "Backup file should exist\n" +title "Backup file should exist" assert_backup_exists -title "Config after logout — empty DEFAULT with comment should remain at the top of the file\n" +title "Config after logout — empty DEFAULT with comment should remain at the top of the file" cat "./home/.databrickscfg" cp "./home/.databrickscfg" "./out.databrickscfg" diff --git a/acceptance/cmd/auth/logout/delete-clears-default/output.txt b/acceptance/cmd/auth/logout/delete-clears-default/output.txt index db8852dcc83..2c20e949d7f 100644 --- a/acceptance/cmd/auth/logout/delete-clears-default/output.txt +++ b/acceptance/cmd/auth/logout/delete-clears-default/output.txt @@ -4,6 +4,7 @@ default_profile = workspace-a === Delete the default profile + >>> [CLI] auth logout --profile workspace-a --auto-approve --delete Deleted profile "workspace-a" with no tokens to clear. @@ -11,6 +12,7 @@ Deleted profile "workspace-a" with no tokens to clear. [__settings__] === Delete a non-default profile + >>> [CLI] auth logout --profile workspace-b --auto-approve --delete Deleted profile "workspace-b" with no tokens to clear. diff --git a/acceptance/cmd/auth/logout/delete-clears-default/script b/acceptance/cmd/auth/logout/delete-clears-default/script index 2028390f4ad..efa7348eec8 100644 --- a/acceptance/cmd/auth/logout/delete-clears-default/script +++ b/acceptance/cmd/auth/logout/delete-clears-default/script @@ -1,18 +1,18 @@ sethome "./home" -title "Initial settings section\n" +title "Initial settings section" cat "./home/.databrickscfg" | grep -A1 __settings__ title "Delete the default profile" trace $CLI auth logout --profile workspace-a --auto-approve --delete -title "Settings after deleting default — default_profile should be cleared\n" +title "Settings after deleting default — default_profile should be cleared" cat "./home/.databrickscfg" | grep -A1 __settings__ title "Delete a non-default profile" trace $CLI auth logout --profile workspace-b --auto-approve --delete -title "Settings after deleting non-default — section should still be present but empty\n" +title "Settings after deleting non-default — section should still be present but empty" cat "./home/.databrickscfg" | grep -A1 __settings__ cp "./home/.databrickscfg" "./out.databrickscfg" diff --git a/acceptance/cmd/auth/logout/delete-pat-token-profile/output.txt b/acceptance/cmd/auth/logout/delete-pat-token-profile/output.txt index 24bfcd3e32b..836f3733051 100644 --- a/acceptance/cmd/auth/logout/delete-pat-token-profile/output.txt +++ b/acceptance/cmd/auth/logout/delete-pat-token-profile/output.txt @@ -8,6 +8,7 @@ host = https://dev.cloud.databricks.com token = dev-pat-token === Logout without --delete — should report no changes for non-U2M profile + >>> [CLI] auth logout --profile dev --auto-approve No tokens to clear for profile "dev". Use --delete to remove it from the config file. @@ -20,6 +21,7 @@ host = https://dev.cloud.databricks.com token = dev-pat-token === Logout with --delete — should delete the profile + >>> [CLI] auth logout --profile dev --auto-approve --delete Deleted profile "dev" with no tokens to clear. diff --git a/acceptance/cmd/auth/logout/delete-pat-token-profile/script b/acceptance/cmd/auth/logout/delete-pat-token-profile/script index fbd2bc18b66..f7cf7398d6f 100644 --- a/acceptance/cmd/auth/logout/delete-pat-token-profile/script +++ b/acceptance/cmd/auth/logout/delete-pat-token-profile/script @@ -1,19 +1,19 @@ sethome "./home" -title "Initial config\n" +title "Initial config" cat "./home/.databrickscfg" title "Logout without --delete — should report no changes for non-U2M profile" trace $CLI auth logout --profile dev --auto-approve -title "Config after logout — profile should be unchanged\n" +title "Config after logout — profile should be unchanged" cat "./home/.databrickscfg" title "Logout with --delete — should delete the profile" trace $CLI auth logout --profile dev --auto-approve --delete -title "Backup file should exist\n" +title "Backup file should exist" assert_backup_exists -title "Config after delete — profile should be removed\n" +title "Config after delete — profile should be removed" cat "./home/.databrickscfg" diff --git a/acceptance/cmd/auth/logout/error-cases/script b/acceptance/cmd/auth/logout/error-cases/script index 08803a732f1..cb1a3861c3d 100644 --- a/acceptance/cmd/auth/logout/error-cases/script +++ b/acceptance/cmd/auth/logout/error-cases/script @@ -1,7 +1,7 @@ sethome "./home" -title "Logout of non-existent profile\n" +title "Logout of non-existent profile" musterr $CLI auth logout --profile nonexistent --auto-approve -title "Logout without --profile in non-interactive mode\n" +title "Logout without --profile in non-interactive mode" musterr $CLI auth logout --auto-approve diff --git a/acceptance/cmd/auth/logout/last-non-default/output.txt b/acceptance/cmd/auth/logout/last-non-default/output.txt index 01828a0a313..8883d0ca3c8 100644 --- a/acceptance/cmd/auth/logout/last-non-default/output.txt +++ b/acceptance/cmd/auth/logout/last-non-default/output.txt @@ -9,6 +9,7 @@ host = [DATABRICKS_URL] auth_type = databricks-cli === Delete the only non-default profile + >>> [CLI] auth logout --profile only-profile --auto-approve --delete Logged out of and deleted profile "only-profile". diff --git a/acceptance/cmd/auth/logout/last-non-default/script b/acceptance/cmd/auth/logout/last-non-default/script index d95069a5552..d1a9aa6b8bc 100644 --- a/acceptance/cmd/auth/logout/last-non-default/script +++ b/acceptance/cmd/auth/logout/last-non-default/script @@ -2,16 +2,16 @@ sethome "./home" envsubst < "./home/.databrickscfg.tmpl" > "./home/.databrickscfg" -title "Initial config\n" +title "Initial config" cat "./home/.databrickscfg" title "Delete the only non-default profile" trace $CLI auth logout --profile only-profile --auto-approve --delete -title "Backup file should exist\n" +title "Backup file should exist" assert_backup_exists -title "Config after logout — DEFAULT section with comment should remain\n" +title "Config after logout — DEFAULT section with comment should remain" cat "./home/.databrickscfg" cp "./home/.databrickscfg" "./out.databrickscfg" diff --git a/acceptance/cmd/auth/logout/ordering-preserved/output.txt b/acceptance/cmd/auth/logout/ordering-preserved/output.txt index 295451562ac..bc6b4791de7 100644 --- a/acceptance/cmd/auth/logout/ordering-preserved/output.txt +++ b/acceptance/cmd/auth/logout/ordering-preserved/output.txt @@ -22,6 +22,7 @@ account_id = account-id auth_type = databricks-cli === Delete first non-default profile (alpha) + >>> [CLI] auth logout --profile alpha --auto-approve --delete Logged out of and deleted profile "alpha". @@ -46,6 +47,7 @@ account_id = account-id auth_type = databricks-cli === Delete last profile (gamma) + >>> [CLI] auth logout --profile gamma --auto-approve --delete Warn: [hostmetadata] failed to fetch host metadata for https://accounts.cloud.databricks.test, will skip for 1m0s Logged out of and deleted profile "gamma". diff --git a/acceptance/cmd/auth/logout/ordering-preserved/script b/acceptance/cmd/auth/logout/ordering-preserved/script index 168657a373c..a484c32912e 100644 --- a/acceptance/cmd/auth/logout/ordering-preserved/script +++ b/acceptance/cmd/auth/logout/ordering-preserved/script @@ -2,22 +2,22 @@ sethome "./home" envsubst < "./home/.databrickscfg.tmpl" > "./home/.databrickscfg" -title "Initial config\n" +title "Initial config" cat "./home/.databrickscfg" title "Delete first non-default profile (alpha)" trace $CLI auth logout --profile alpha --auto-approve --delete -title "Backup file should exist\n" +title "Backup file should exist" assert_backup_exists -title "Config after deleting alpha\n" +title "Config after deleting alpha" cat "./home/.databrickscfg" title "Delete last profile (gamma)" trace $CLI auth logout --profile gamma --auto-approve --delete -title "Config after deleting gamma\n" +title "Config after deleting gamma" cat "./home/.databrickscfg" cp "./home/.databrickscfg" "./out.databrickscfg" diff --git a/acceptance/cmd/auth/logout/pending-device-code-login/output.txt b/acceptance/cmd/auth/logout/pending-device-code-login/output.txt index 2b000f667ce..f7bd86318e6 100644 --- a/acceptance/cmd/auth/logout/pending-device-code-login/output.txt +++ b/acceptance/cmd/auth/logout/pending-device-code-login/output.txt @@ -1,5 +1,6 @@ === Logout and delete dev + >>> [CLI] auth logout --profile dev --auto-approve --delete Logged out of and deleted profile "dev". @@ -13,6 +14,7 @@ Logged out of and deleted profile "dev". ] === Logout staging + >>> [CLI] auth logout --profile staging --auto-approve Logged out of profile "staging". Use --delete to also remove it from the config file. @@ -22,6 +24,7 @@ Logged out of profile "staging". Use --delete to also remove it from the config ] === Logout pat, a profile that auth login did not create + >>> [CLI] auth logout --profile pat --auto-approve No tokens to clear for profile "pat". Use --delete to remove it from the config file. diff --git a/acceptance/cmd/auth/logout/pending-device-code-login/script b/acceptance/cmd/auth/logout/pending-device-code-login/script index 32a2cc1d427..108afb58459 100644 --- a/acceptance/cmd/auth/logout/pending-device-code-login/script +++ b/acceptance/cmd/auth/logout/pending-device-code-login/script @@ -33,17 +33,17 @@ EOF title "Logout and delete dev" trace $CLI auth logout --profile dev --auto-approve --delete -title "Token cache keys after logout — the host keys are kept, staging shares them\n" +title "Token cache keys after logout — the host keys are kept, staging shares them" jq -S '.tokens | keys' "./home/.databricks/token-cache.json" title "Logout staging" trace $CLI auth logout --profile staging --auto-approve -title "Token cache keys after logout — no profile uses the host anymore\n" +title "Token cache keys after logout — no profile uses the host anymore" jq -S '.tokens | keys' "./home/.databricks/token-cache.json" title "Logout pat, a profile that auth login did not create" trace $CLI auth logout --profile pat --auto-approve -title "Token cache keys after logout — the pending login of pat is removed\n" +title "Token cache keys after logout — the pending login of pat is removed" jq -S '.tokens | keys' "./home/.databricks/token-cache.json" diff --git a/acceptance/cmd/auth/logout/stale-account-id-workspace-host/output.txt b/acceptance/cmd/auth/logout/stale-account-id-workspace-host/output.txt index 8c3218ad06e..239d456c0c1 100644 --- a/acceptance/cmd/auth/logout/stale-account-id-workspace-host/output.txt +++ b/acceptance/cmd/auth/logout/stale-account-id-workspace-host/output.txt @@ -1,5 +1,6 @@ === Profiles before logout — logfood should be valid + >>> [CLI] auth profiles Name Host Valid logfood (Default) [DATABRICKS_URL] YES @@ -11,6 +12,7 @@ logfood (Default) [DATABRICKS_URL] YES ] === Logout without --delete + >>> [CLI] auth logout --profile logfood --auto-approve Logged out of profile "logfood". Use --delete to also remove it from the config file. @@ -30,10 +32,12 @@ default_profile = logfood [] === Profiles after logout — logfood should be invalid + >>> [CLI] auth profiles Name Host Valid logfood (Default) [DATABRICKS_URL] NO === Logged out profile should no longer return a token + >>> musterr [CLI] auth token --profile logfood Error: cache: databricks OAuth is not configured for this host. no cached credentials; run `databricks auth login` to sign in diff --git a/acceptance/cmd/auth/logout/stale-account-id-workspace-host/script b/acceptance/cmd/auth/logout/stale-account-id-workspace-host/script index bd477db2e29..a7060c6578f 100644 --- a/acceptance/cmd/auth/logout/stale-account-id-workspace-host/script +++ b/acceptance/cmd/auth/logout/stale-account-id-workspace-host/script @@ -33,16 +33,16 @@ EOF title "Profiles before logout — logfood should be valid" trace $CLI auth profiles -title "Token cache keys before logout\n" +title "Token cache keys before logout" jq -S '.tokens | keys' "./home/.databricks/token-cache.json" title "Logout without --delete" trace $CLI auth logout --profile logfood --auto-approve -title "Config after logout — profile should still exist\n" +title "Config after logout — profile should still exist" cat "./home/.databrickscfg" -title "Token cache keys after logout — both entries should be removed\n" +title "Token cache keys after logout — both entries should be removed" jq -S '.tokens | keys' "./home/.databricks/token-cache.json" title "Profiles after logout — logfood should be invalid" diff --git a/acceptance/cmd/auth/logout/token-only-shared-host/output.txt b/acceptance/cmd/auth/logout/token-only-shared-host/output.txt index a608a51dd64..d6944f4527e 100644 --- a/acceptance/cmd/auth/logout/token-only-shared-host/output.txt +++ b/acceptance/cmd/auth/logout/token-only-shared-host/output.txt @@ -7,6 +7,7 @@ ] === Logout dev without --delete + >>> [CLI] auth logout --profile dev --auto-approve Logged out of profile "dev". Use --delete to also remove it from the config file. diff --git a/acceptance/cmd/auth/logout/token-only-shared-host/script b/acceptance/cmd/auth/logout/token-only-shared-host/script index da37ff6dbdb..11e591802b6 100644 --- a/acceptance/cmd/auth/logout/token-only-shared-host/script +++ b/acceptance/cmd/auth/logout/token-only-shared-host/script @@ -34,14 +34,14 @@ cat > "./home/.databricks/token-cache.json" <>> [CLI] auth logout --profile dev --auto-approve Warn: [hostmetadata] failed to fetch host metadata for https://accounts.cloud.databricks.test, will skip for 1m0s Logged out of profile "dev". Use --delete to also remove it from the config file. diff --git a/acceptance/cmd/auth/logout/token-only/script b/acceptance/cmd/auth/logout/token-only/script index bce46a7986b..ea004849f7b 100644 --- a/acceptance/cmd/auth/logout/token-only/script +++ b/acceptance/cmd/auth/logout/token-only/script @@ -1,14 +1,14 @@ sethome "./home" # home/.databrickscfg and home/.databricks/token-cache.json are checked-in inputs. -title "Token cache keys before logout\n" +title "Token cache keys before logout" jq -S '.tokens | keys' "./home/.databricks/token-cache.json" title "Logout without --delete" trace $CLI auth logout --profile dev --auto-approve -title "Config after logout — profile should still exist\n" +title "Config after logout — profile should still exist" cat "./home/.databrickscfg" -title "Token cache keys after logout — both entries should be removed\n" +title "Token cache keys after logout — both entries should be removed" jq -S '.tokens | keys' "./home/.databricks/token-cache.json" diff --git a/acceptance/cmd/auth/profiles/script b/acceptance/cmd/auth/profiles/script index d8342212d37..61ba384c9fb 100644 --- a/acceptance/cmd/auth/profiles/script +++ b/acceptance/cmd/auth/profiles/script @@ -1,5 +1,5 @@ sethome "./home" # home/.databrickscfg is a checked-in input with profiles including one with workspace_id. -title "Profiles with workspace_id (JSON output)\n" +title "Profiles with workspace_id (JSON output)" $CLI auth profiles --skip-validate --output json diff --git a/acceptance/cmd/auth/profiles/spog-account/output.txt b/acceptance/cmd/auth/profiles/spog-account/output.txt index 7ec701d700e..7821e4cfa50 100644 --- a/acceptance/cmd/auth/profiles/spog-account/output.txt +++ b/acceptance/cmd/auth/profiles/spog-account/output.txt @@ -1,5 +1,6 @@ === SPOG account profile should be valid + >>> [CLI] auth profiles --output json { "profiles": [ diff --git a/acceptance/cmd/auth/storage-modes/env-overrides-config/script b/acceptance/cmd/auth/storage-modes/env-overrides-config/script index ddab417910d..14530bddfd1 100644 --- a/acceptance/cmd/auth/storage-modes/env-overrides-config/script +++ b/acceptance/cmd/auth/storage-modes/env-overrides-config/script @@ -2,7 +2,7 @@ export DATABRICKS_AUTH_STORAGE=plaintext # home/.databrickscfg ([__settings__] auth_storage = secure) and # home/.databricks/token-cache.json are checked-in inputs. -title "Token cache keys before logout\n" +title "Token cache keys before logout" jq -S '.tokens | keys' "./home/.databricks/token-cache.json" # Env var = plaintext must beat [__settings__] auth_storage = secure. @@ -10,5 +10,5 @@ jq -S '.tokens | keys' "./home/.databricks/token-cache.json" # were ignored, the CLI would try the keyring and leave the file cache alone. trace $CLI auth logout --profile dev --auto-approve -title "Token cache keys after logout (should be empty)\n" +title "Token cache keys after logout (should be empty)" jq -S '.tokens | keys' "./home/.databricks/token-cache.json" diff --git a/acceptance/cmd/auth/storage-modes/plaintext-env-default/script b/acceptance/cmd/auth/storage-modes/plaintext-env-default/script index d59b26e2195..78a2bb634f0 100644 --- a/acceptance/cmd/auth/storage-modes/plaintext-env-default/script +++ b/acceptance/cmd/auth/storage-modes/plaintext-env-default/script @@ -1,10 +1,10 @@ export DATABRICKS_AUTH_STORAGE=plaintext # home/.databrickscfg and home/.databricks/token-cache.json are checked-in inputs. -title "Token cache keys before logout\n" +title "Token cache keys before logout" jq -S '.tokens | keys' "./home/.databricks/token-cache.json" trace $CLI auth logout --profile dev --auto-approve -title "Token cache keys after logout (should be empty)\n" +title "Token cache keys after logout (should be empty)" jq -S '.tokens | keys' "./home/.databricks/token-cache.json" diff --git a/acceptance/cmd/auth/switch/nominal/script b/acceptance/cmd/auth/switch/nominal/script index b10cd6c03e7..a96311f5cce 100644 --- a/acceptance/cmd/auth/switch/nominal/script +++ b/acceptance/cmd/auth/switch/nominal/script @@ -3,22 +3,22 @@ sethome "./home" # Two profiles without a [__settings__] section. envsubst < "./home/.databrickscfg.tmpl" > "./home/.databrickscfg" -title "Switch to profile-a\n" +title "Switch to profile-a" trace $CLI auth switch --profile profile-a -title "Settings after first switch\n" +title "Settings after first switch" cat "./home/.databrickscfg" | grep -A1 __settings__ -title "Profiles after first switch\n" +title "Profiles after first switch" trace $CLI auth profiles --skip-validate -title "Switch to profile-b\n" +title "Switch to profile-b" trace $CLI auth switch --profile profile-b -title "Settings after second switch\n" +title "Settings after second switch" cat "./home/.databrickscfg" | grep -A1 __settings__ -title "Profiles after second switch\n" +title "Profiles after second switch" trace $CLI auth profiles --skip-validate # Track the final .databrickscfg to surface changes. diff --git a/acceptance/cmd/auth/token/default-profile/script b/acceptance/cmd/auth/token/default-profile/script index f5de6231901..0eda54a8a39 100644 --- a/acceptance/cmd/auth/token/default-profile/script +++ b/acceptance/cmd/auth/token/default-profile/script @@ -19,10 +19,10 @@ auth_type = databricks-cli default_profile = myprofile ENDCFG -title "default_profile is honored when no args, --profile, or env var\n" +title "default_profile is honored when no args, --profile, or env var" trace errcode $CLI auth token -title "default_profile pointing at a missing profile falls through to picker\n" +title "default_profile pointing at a missing profile falls through to picker" cat > "./home/.databrickscfg" <<'ENDCFG' [myprofile] host = https://myworkspace.test diff --git a/acceptance/cmd/configure/clears-oauth-on-pat/output.txt b/acceptance/cmd/configure/clears-oauth-on-pat/output.txt index 6142be378ed..e99863d6a55 100644 --- a/acceptance/cmd/configure/clears-oauth-on-pat/output.txt +++ b/acceptance/cmd/configure/clears-oauth-on-pat/output.txt @@ -9,6 +9,7 @@ account_id = acc-123 workspace_id = ws-456 === Run configure with PAT token + >>> [CLI] configure --token --host https://host === Profile after PAT configure diff --git a/acceptance/cmd/configure/clears-oauth-on-pat/script b/acceptance/cmd/configure/clears-oauth-on-pat/script index 6b00490f79a..93efe480cb9 100644 --- a/acceptance/cmd/configure/clears-oauth-on-pat/script +++ b/acceptance/cmd/configure/clears-oauth-on-pat/script @@ -2,11 +2,11 @@ sethome "./home" # home/.databrickscfg is a checked-in input: a profile with OAuth metadata and # unified host fields (as if `auth login` was previously run against a unified host). -title "Initial profile (OAuth with unified host)\n" +title "Initial profile (OAuth with unified host)" cat "./home/.databrickscfg" title "Run configure with PAT token" echo "new-token" | trace $CLI configure --token --host https://host -title "Profile after PAT configure\n" +title "Profile after PAT configure" cat "./home/.databrickscfg" diff --git a/acceptance/cmd/configure/clears-serverless-when-cluster-from-env/output.txt b/acceptance/cmd/configure/clears-serverless-when-cluster-from-env/output.txt index a0ef365e1dc..fe068da62cf 100644 --- a/acceptance/cmd/configure/clears-serverless-when-cluster-from-env/output.txt +++ b/acceptance/cmd/configure/clears-serverless-when-cluster-from-env/output.txt @@ -5,6 +5,7 @@ host = https://host serverless_compute_id = auto === Run configure with cluster from env var + >>> [CLI] configure --token --host https://host === Profile after configure (serverless should be cleared) diff --git a/acceptance/cmd/configure/clears-serverless-when-cluster-from-env/script b/acceptance/cmd/configure/clears-serverless-when-cluster-from-env/script index dcee6ef1209..0be20f66791 100644 --- a/acceptance/cmd/configure/clears-serverless-when-cluster-from-env/script +++ b/acceptance/cmd/configure/clears-serverless-when-cluster-from-env/script @@ -1,7 +1,7 @@ sethome "./home" # home/.databrickscfg is a checked-in input: a profile with serverless_compute_id. -title "Initial profile with serverless_compute_id\n" +title "Initial profile with serverless_compute_id" cat "./home/.databrickscfg" # Set cluster ID via env var (not via --configure-cluster flag). @@ -10,5 +10,5 @@ export DATABRICKS_CLUSTER_ID=env-cluster-789 title "Run configure with cluster from env var" echo "new-token" | trace $CLI configure --token --host https://host -title "Profile after configure (serverless should be cleared)\n" +title "Profile after configure (serverless should be cleared)" cat "./home/.databrickscfg" diff --git a/acceptance/cmd/fs/cp/file-to-nonexistent-dir/output.txt b/acceptance/cmd/fs/cp/file-to-nonexistent-dir/output.txt index 9122e25d50e..155194bc619 100644 --- a/acceptance/cmd/fs/cp/file-to-nonexistent-dir/output.txt +++ b/acceptance/cmd/fs/cp/file-to-nonexistent-dir/output.txt @@ -1,5 +1,6 @@ === foo.txt -> local + >>> [CLI] fs cp foo.txt local/existing-from-local/ foo.txt -> local/existing-from-local/foo.txt @@ -9,6 +10,7 @@ Error: no such directory: local/nonexistent/mydir Exit code: 1 === foo.txt -> dbfs:/tmp/fs-cp-test-[UNIQUE_NAME] + >>> [CLI] fs cp foo.txt dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/existing-from-local/ foo.txt -> dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/existing-from-local/foo.txt @@ -18,6 +20,7 @@ Error: no such directory: /tmp/fs-cp-test-[UNIQUE_NAME]/nonexistent Exit code: 1 === foo.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME] + >>> [CLI] fs cp foo.txt dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/existing-from-local/ foo.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/existing-from-local/foo.txt @@ -27,6 +30,7 @@ Error: no such directory: /Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/nonexis Exit code: 1 === dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> local + >>> [CLI] fs cp dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt local/existing-from-dbfs/ dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> local/existing-from-dbfs/foo.txt @@ -36,6 +40,7 @@ Error: no such directory: local/nonexistent/mydir Exit code: 1 === dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> dbfs:/tmp/fs-cp-test-[UNIQUE_NAME] + >>> [CLI] fs cp dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/existing-from-dbfs/ dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/existing-from-dbfs/foo.txt @@ -45,6 +50,7 @@ Error: no such directory: /tmp/fs-cp-test-[UNIQUE_NAME]/nonexistent Exit code: 1 === dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME] + >>> [CLI] fs cp dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/existing-from-dbfs/ dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/existing-from-dbfs/foo.txt @@ -54,6 +60,7 @@ Error: no such directory: /Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/nonexis Exit code: 1 === dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> local + >>> [CLI] fs cp dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt local/existing-from-volume/ dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> local/existing-from-volume/foo.txt @@ -63,6 +70,7 @@ Error: no such directory: local/nonexistent/mydir Exit code: 1 === dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> dbfs:/tmp/fs-cp-test-[UNIQUE_NAME] + >>> [CLI] fs cp dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/existing-from-volume/ dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> dbfs:/tmp/fs-cp-test-[UNIQUE_NAME]/existing-from-volume/foo.txt @@ -72,6 +80,7 @@ Error: no such directory: /tmp/fs-cp-test-[UNIQUE_NAME]/nonexistent Exit code: 1 === dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME] + >>> [CLI] fs cp dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/existing-from-volume/ dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/src/foo.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/existing-from-volume/foo.txt @@ -81,6 +90,7 @@ Error: no such directory: /Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/nonexis Exit code: 1 === Copies landed in the existing directories + >>> cat local/existing-from-local/foo.txt abc diff --git a/acceptance/cmd/fs/dbfs/output.txt b/acceptance/cmd/fs/dbfs/output.txt index da4966b8648..108aad29f69 100644 --- a/acceptance/cmd/fs/dbfs/output.txt +++ b/acceptance/cmd/fs/dbfs/output.txt @@ -1,15 +1,18 @@ === Create a directory and write a file in it + >>> [CLI] fs mkdir dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir >>> [CLI] fs cp big.bin dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir/big.bin big.bin -> dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir/big.bin === Read the file back + >>> cmp big.bin big.back contents match === Writing over an existing file needs --overwrite + >>> [CLI] fs cp big.bin dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir/big.bin big.bin -> dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir/big.bin (skipped; already exists) @@ -17,12 +20,14 @@ big.bin -> dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir/big.bin (skipped; already exists) big.bin -> dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir/big.bin === Writing into a directory that does not exist fails + >>> [CLI] fs cp big.bin dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/missing/big.bin Error: no such directory: /tmp/fs-dbfs-[UNIQUE_NAME]/missing Exit code: 1 === List a directory, then a file + >>> [CLI] fs ls dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir big.bin @@ -32,12 +37,14 @@ Error: not a directory: /tmp/fs-dbfs-[UNIQUE_NAME]/dir/big.bin Exit code: 1 === cat a directory fails + >>> [CLI] fs cat dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir Error: not a file: /tmp/fs-dbfs-[UNIQUE_NAME]/dir Exit code: 1 === rm a non-empty directory needs -r + >>> [CLI] fs rm dbfs:/tmp/fs-dbfs-[UNIQUE_NAME]/dir Error: directory not empty: /tmp/fs-dbfs-[UNIQUE_NAME]/dir diff --git a/acceptance/cmd/liteswap-target/output.txt b/acceptance/cmd/liteswap-target/output.txt index 9f3a29c49eb..b8b59b78545 100644 --- a/acceptance/cmd/liteswap-target/output.txt +++ b/acceptance/cmd/liteswap-target/output.txt @@ -1,5 +1,6 @@ === Workspace command + >>> [CLI] clusters list ID Name State 4321 some-test-cluster @@ -17,6 +18,7 @@ ID Name State } === Account command + >>> [CLI] account network-policies list-network-policies-rpc [ @@ -34,6 +36,7 @@ ID Name State } === Bundle command + >>> [CLI] bundle validate Name: liteswap-target Target: default @@ -55,6 +58,7 @@ Validation OK! } === No header when unset + >>> [CLI] clusters list ID Name State 4321 some-test-cluster diff --git a/acceptance/cmd/patchwhl/output.txt b/acceptance/cmd/patchwhl/output.txt index d6d9972aa2b..f8fa10cba78 100644 --- a/acceptance/cmd/patchwhl/output.txt +++ b/acceptance/cmd/patchwhl/output.txt @@ -1,5 +1,6 @@ === Test prebuilt wheel: + >>> setmtime.py [TIMESTAMP] my_test_code-0.0.1-py3-none-any.whl >>> [CLI] selftest patchwhl my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/cmd/psql/argument-errors/output.txt b/acceptance/cmd/psql/argument-errors/output.txt index 35da5961dec..a55480ea090 100644 --- a/acceptance/cmd/psql/argument-errors/output.txt +++ b/acceptance/cmd/psql/argument-errors/output.txt @@ -1,33 +1,41 @@ === Provisioned instance with --project flag should error: + >>> musterr [CLI] psql my-instance --project foo Error: cannot use --project, --branch, or --endpoint flags with a provisioned instance name === Provisioned instance with --branch flag should error: + >>> musterr [CLI] psql my-instance --branch main Error: cannot use --project, --branch, or --endpoint flags with a provisioned instance name === Provisioned instance with --endpoint flag should error: + >>> musterr [CLI] psql my-instance --endpoint primary Error: cannot use --project, --branch, or --endpoint flags with a provisioned instance name === Full path with conflicting --project flag should error: + >>> musterr [CLI] psql projects/my-project/branches/main/endpoints/primary --project other Error: --project flag conflicts with project in path: other vs my-project === Full path with conflicting --branch flag should error: + >>> musterr [CLI] psql projects/my-project/branches/main/endpoints/primary --branch other Error: --branch flag conflicts with branch in path: other vs main === Full path with conflicting --endpoint flag should error: + >>> musterr [CLI] psql projects/my-project/branches/main/endpoints/primary --endpoint other Error: --endpoint flag conflicts with endpoint in path: other vs primary === Partial path (project+branch) with conflicting --branch flag should error: + >>> musterr [CLI] psql projects/my-project/branches/main --branch other Error: --branch flag conflicts with branch in path: other vs main === Partial path (project only) with --branch flag should supplement: + >>> [CLI] psql projects/my-project --branch main --max-retries 0 Project: My Project Branch: main @@ -38,6 +46,7 @@ PGPASSWORD=postgres-secret-token PGSSLMODE=require === Partial path (project+branch) with --endpoint flag should supplement: + >>> [CLI] psql projects/my-project/branches/dev --endpoint custom --max-retries 0 Project: My Project Branch: dev @@ -48,29 +57,36 @@ PGPASSWORD=postgres-secret-token PGSSLMODE=require === Invalid path with just 'projects/' should error: + >>> musterr [CLI] psql projects/ Error: invalid resource path: missing project ID === Invalid path with missing branch ID should error: + >>> musterr [CLI] psql projects/my-project/branches/ Error: invalid resource path: missing branch ID === Invalid path with missing endpoint ID should error: + >>> musterr [CLI] psql projects/my-project/branches/main/endpoints/ Error: invalid resource path: missing endpoint ID === Provisioned flag with --project should error: + >>> musterr [CLI] psql --provisioned --project foo Error: cannot use --project, --branch, or --endpoint flags with --provisioned === Provisioned flag with autoscaling path should error: + >>> musterr [CLI] psql --provisioned projects/my-project/branches/main/endpoints/primary Error: cannot use --provisioned flag with an autoscaling resource path === Autoscaling flag with provisioned instance should error: + >>> musterr [CLI] psql --autoscaling my-instance Error: cannot use --autoscaling flag with a provisioned instance name === Provisioned and autoscaling flags are mutually exclusive: + >>> musterr [CLI] psql --provisioned --autoscaling Error: if any flags in the group [provisioned autoscaling] are set none of the others can be; [autoscaling provisioned] were all set diff --git a/acceptance/cmd/psql/completions/script b/acceptance/cmd/psql/completions/script index b521c171224..6f859d8e17b 100644 --- a/acceptance/cmd/psql/completions/script +++ b/acceptance/cmd/psql/completions/script @@ -1,2 +1,2 @@ -title "Command should show instances names in autocomplete:\n" +title "Command should show instances names in autocomplete:" $CLI __complete psql "" diff --git a/acceptance/cmd/psql/failing-connection/output.txt b/acceptance/cmd/psql/failing-connection/output.txt index 1de1feabd0e..b6c58e60f96 100644 --- a/acceptance/cmd/psql/failing-connection/output.txt +++ b/acceptance/cmd/psql/failing-connection/output.txt @@ -1,5 +1,6 @@ === Command should use default number of retries: + >>> musterr [CLI] psql my-database -- --dbname=db1 -p 3000 Instance: my-database Connecting to database instance... @@ -16,6 +17,7 @@ Connection failed with retryable error: connection failed: psql exited with code Error: failed to connect after 3 attempts, last error: connection failed: psql exited with code 2 === Command should use custom number of retries: + >>> musterr [CLI] psql my-database --max-retries 5 -- --dbname=db1 -p 3000 Instance: my-database Connecting to database instance... @@ -40,6 +42,7 @@ Connection failed with retryable error: connection failed: psql exited with code Error: failed to connect after 5 attempts, last error: connection failed: psql exited with code 2 === Command should not use retries: + >>> musterr [CLI] psql my-database --max-retries 0 -- --dbname=db1 -p 3000 Instance: my-database Connecting to database instance... diff --git a/acceptance/cmd/psql/not-available/output.txt b/acceptance/cmd/psql/not-available/output.txt index 0355810a7de..5bd4a62362b 100644 --- a/acceptance/cmd/psql/not-available/output.txt +++ b/acceptance/cmd/psql/not-available/output.txt @@ -1,5 +1,6 @@ === Command should exit if the instance is not available + >>> musterr [CLI] psql database-starting Instance: database-starting Instance status: STARTING diff --git a/acceptance/cmd/psql/postgres/output.txt b/acceptance/cmd/psql/postgres/output.txt index 5269553a0ce..c3f9250ea64 100644 --- a/acceptance/cmd/psql/postgres/output.txt +++ b/acceptance/cmd/psql/postgres/output.txt @@ -1,11 +1,13 @@ === Test should use a fake psql command: + >>> psql --version echo-arguments.sh was called with the following arguments: --version PGPASSWORD= PGSSLMODE= === Full endpoint path should connect to the endpoint: + >>> [CLI] psql projects/my-project/branches/main/endpoints/primary Project: My Project Branch: main @@ -16,6 +18,7 @@ PGPASSWORD=postgres-secret-token PGSSLMODE=require === Project flag with auto-selection should connect: + >>> [CLI] psql --project auto-project Project: Auto Project Branch: default-branch @@ -26,6 +29,7 @@ PGPASSWORD=postgres-secret-token PGSSLMODE=require === Project and branch flags should connect: + >>> [CLI] psql --project explicit-project --branch explicit-branch Project: Explicit Project Branch: explicit-branch @@ -36,6 +40,7 @@ PGPASSWORD=postgres-secret-token PGSSLMODE=require === Project, branch, and endpoint flags should connect: + >>> [CLI] psql --project explicit-project --branch explicit-branch --endpoint main-ep Project: Explicit Project Branch: explicit-branch @@ -46,6 +51,7 @@ PGPASSWORD=postgres-secret-token PGSSLMODE=require === Endpoint in INIT state should fail: + >>> musterr [CLI] psql projects/init-project/branches/main/endpoints/init-ep Project: Init Project Branch: main @@ -53,14 +59,17 @@ Endpoint: init-ep Error: endpoint is not ready for accepting connections === Branch flag without project should fail: + >>> musterr [CLI] psql --branch some-branch Error: --project is required when using --branch or --endpoint === Endpoint flag without project should fail: + >>> musterr [CLI] psql --endpoint some-endpoint Error: --project is required when using --branch or --endpoint === Extra arguments should be passed through with full path: + >>> [CLI] psql projects/my-project/branches/main/endpoints/primary -- -c SELECT 1 Project: My Project Branch: main @@ -71,6 +80,7 @@ PGPASSWORD=postgres-secret-token PGSSLMODE=require === Extra arguments should be passed through with project flag: + >>> [CLI] psql --project auto-project -- --echo-all -d my-db Project: Auto Project Branch: default-branch @@ -81,6 +91,7 @@ PGPASSWORD=postgres-secret-token PGSSLMODE=require === No retries with max-retries 0: + >>> [CLI] psql --project auto-project --max-retries 0 Project: Auto Project Branch: default-branch diff --git a/acceptance/cmd/psql/simple/output.txt b/acceptance/cmd/psql/simple/output.txt index d2d9a983cd7..603bfcf502e 100644 --- a/acceptance/cmd/psql/simple/output.txt +++ b/acceptance/cmd/psql/simple/output.txt @@ -1,19 +1,23 @@ === Test should use a fake psql command: + >>> psql --version echo-arguments.sh was called with the following arguments: --version PGPASSWORD= PGSSLMODE= === Command should require instance name: + >>> musterr [CLI] psql Error: no Lakebase databases found in workspace === Command should require instance name even if extra arguments are provided: + >>> musterr [CLI] psql -- --dbname=my-database Error: no Lakebase databases found in workspace === Command should use default db name if it is not specified: + >>> [CLI] psql my-database Instance: my-database Connecting to database instance... @@ -22,6 +26,7 @@ PGPASSWORD=my-secret-token PGSSLMODE=require === Command should by-pass extra arguments to psql: + >>> [CLI] psql my-database -- -c SELECT * FROM my_table --echo-all Instance: my-database Connecting to database instance... @@ -30,6 +35,7 @@ PGPASSWORD=my-secret-token PGSSLMODE=require === Command should use the db name from extra arguments when specified: + >>> [CLI] psql my-database -- --dbname=db1 Instance: my-database Connecting to database instance... @@ -45,6 +51,7 @@ PGPASSWORD=my-secret-token PGSSLMODE=require === Command should use the port from extra arguments when specified: + >>> [CLI] psql my-database -- --dbname=db1 -p 3000 Instance: my-database Connecting to database instance... @@ -60,6 +67,7 @@ PGPASSWORD=my-secret-token PGSSLMODE=require === Command should not use retries if max-retries is set to 0: + >>> [CLI] psql my-database --max-retries 0 -- --dbname=db1 -p 3000 Instance: my-database Connecting to database instance... diff --git a/acceptance/cmd/version/script b/acceptance/cmd/version/script index d4e36422ed0..85f4b5bbd3d 100644 --- a/acceptance/cmd/version/script +++ b/acceptance/cmd/version/script @@ -1,10 +1,10 @@ -title "version stays lightweight\n" +title "version stays lightweight" trace $CLI version # The acceptance binary is a development build, so the check short-circuits # without contacting GitHub. This exercises command wiring and output rendering. -title "version --check\n" +title "version --check" trace $CLI version --check -title "version --check --output json\n" +title "version --check --output json" trace $CLI version --check --output json diff --git a/acceptance/cmd/workspace/apps/output.txt b/acceptance/cmd/workspace/apps/output.txt index f4ea28d062a..17b8b051ffe 100644 --- a/acceptance/cmd/workspace/apps/output.txt +++ b/acceptance/cmd/workspace/apps/output.txt @@ -1,5 +1,6 @@ === Apps create with correct input + >>> [CLI] apps create --json @input.json { "active_deployment": { @@ -42,6 +43,7 @@ } === Apps update with correct input + >>> [CLI] apps update test-name --json @input.json { "active_deployment": { @@ -84,6 +86,7 @@ } === Apps update with missing parameter + >>> [CLI] apps update --json @input.json Error: accepts 1 arg(s), received 0 diff --git a/acceptance/cmd/workspace/apps/run-local-node/script b/acceptance/cmd/workspace/apps/run-local-node/script index 2bd2238c54b..72fe2ba2f5f 100755 --- a/acceptance/cmd/workspace/apps/run-local-node/script +++ b/acceptance/cmd/workspace/apps/run-local-node/script @@ -6,7 +6,7 @@ read -r APP_PORT PROXY_PORT DEBUG_PORT PREPARE_PORT <<< "$(free_port.py 4)" # The app and the proxy start asynchronously; allow up to 15s for them to come up. export RETRY_MAX_ATTEMPTS=30 -title "Prepare environment runs the build script, and the entry point overrides app.yml\n" +title "Prepare environment runs the build script, and the entry point overrides app.yml" # npm reaches no registry here: nothing to install, and app/.npmrc turns off its update check. # The build script runs only if the install succeeded, so the first grep covers both halves. trace $CLI apps run-local --prepare-environment --entry-point test.yml --port "$PREPARE_PORT" &> ../LOG.prepare @@ -29,15 +29,15 @@ PID=$! trap cleanup EXIT cd .. -title "Proxy forwards to the app\n" +title "Proxy forwards to the app" trace retry --until "Hello From App" curl -sS "http://127.0.0.1:$PROXY_PORT" -title "Debug mode reports the debugger port, and node listens on it\n" +title "Debug mode reports the debugger port, and node listens on it" # The CLI prints this only after the proxy accepts connections, so the curl above can beat it. # node's line below needs no retry: it is written before the app listens, so the curl proves it. trace retry grep -x "To debug your app, attach a debugger to port $DEBUG_PORT" LOG.run trace grep "Debugger listening on ws://127.0.0.1:$DEBUG_PORT/" LOG.run -title "Shutting the app down\n" +title "Shutting the app down" trace curl -sS -o /dev/null "http://127.0.0.1:$PROXY_PORT/shutdown" wait $PID diff --git a/acceptance/cmd/workspace/apps/run-local/script b/acceptance/cmd/workspace/apps/run-local/script index f16f7db1d78..7dbb6df021d 100755 --- a/acceptance/cmd/workspace/apps/run-local/script +++ b/acceptance/cmd/workspace/apps/run-local/script @@ -6,10 +6,10 @@ read -r APP_PORT PROXY_PORT ENTRY_POINT_PORT <<< "$(free_port.py 3)" # The app and the proxy start asynchronously; allow up to 15s for them to come up. export RETRY_MAX_ATTEMPTS=30 -title "Env var with valueFrom cannot be resolved locally\n" +title "Env var with valueFrom cannot be resolved locally" trace errcode $CLI apps run-local --entry-point value-from.yml 2>&1 -title "Entry point overrides the command from app.yml\n" +title "Entry point overrides the command from app.yml" trace $CLI apps run-local --entry-point test.yml --port "$ENTRY_POINT_PORT" &> ../LOG.entry-point trace grep -x "Running command: python -c print('Hello, world')" ../LOG.entry-point trace grep -x "Hello, world" ../LOG.entry-point @@ -30,15 +30,15 @@ PID=$! trap cleanup EXIT cd .. -title "Proxy forwards to the app and injects the identity headers\n" +title "Proxy forwards to the app and injects the identity headers" trace retry --until X-Forwarded-Email curl -sS "http://127.0.0.1:$PROXY_PORT" -title "Proxy URL is reported on the port that was asked for\n" +title "Proxy URL is reported on the port that was asked for" trace grep -x "To access your app go to http://localhost:$PROXY_PORT" LOG.run -title "App picked up the env var from app.yml\n" +title "App picked up the env var from app.yml" trace grep -x "Python app has started with: test" LOG.run -title "Shutting the app down\n" +title "Shutting the app down" trace curl -sS -o /dev/null "http://127.0.0.1:$PROXY_PORT/shutdown" wait $PID diff --git a/acceptance/cmd/workspace/arg-validation/script b/acceptance/cmd/workspace/arg-validation/script index 064065c87ca..e53f8a0bfac 100644 --- a/acceptance/cmd/workspace/arg-validation/script +++ b/acceptance/cmd/workspace/arg-validation/script @@ -1,8 +1,8 @@ # Generated commands validate their input before issuing any request, so these # cases need no workspace. -title "clusters create requires a positional arg (or --json)\n" +title "clusters create requires a positional arg (or --json)" trace musterr $CLI clusters create -title "alerts-legacy create only accepts JSON input\n" +title "alerts-legacy create only accepts JSON input" trace musterr $CLI alerts-legacy create diff --git a/acceptance/cmd/workspace/clusters-list/script b/acceptance/cmd/workspace/clusters-list/script index eee3f5887aa..2c8d0c70c30 100644 --- a/acceptance/cmd/workspace/clusters-list/script +++ b/acceptance/cmd/workspace/clusters-list/script @@ -1,2 +1,2 @@ -title "clusters list renders the ID/Name/State columns\n" +title "clusters list renders the ID/Name/State columns" trace $CLI clusters list diff --git a/acceptance/cmd/workspace/postgres/create-branch/output.txt b/acceptance/cmd/workspace/postgres/create-branch/output.txt index 77a585bc3d0..c7b7853b7e0 100644 --- a/acceptance/cmd/workspace/postgres/create-branch/output.txt +++ b/acceptance/cmd/workspace/postgres/create-branch/output.txt @@ -1,16 +1,28 @@ === Set up the parent project + === create-branch --ttl: API (604800s), Go (168h) and day (7d) forms all go on the wire as seconds + === create-branch --no-expiry + === create-branch --ttl combined with a --json spec: the flag's ttl survives the json merge + === the pre-existing flags still work alongside --ttl: --name (body), --replace-existing (query), --timeout (wait) + === the --no-wait path still works alongside --no-expiry + === An expiration is required: no flag fails before any API call (no request recorded) + === Conflicting flags fail before any API call (no request recorded) + === --no-expiry with a --json that also sets spec.no_expiry is a conflict, not a silent override + === Malformed --json fails client-side + === An empty spec.ttl in --json fails client-side -=== Recorded create-branch request bodies{ + +=== Recorded create-branch request bodies +{ "method": "POST", "path": "/api/2.0/postgres/projects/acc-proj/branches", "q": { diff --git a/acceptance/cmd/workspace/put-secret/script b/acceptance/cmd/workspace/put-secret/script index a4f41d32b1e..c504a3de5ce 100644 --- a/acceptance/cmd/workspace/put-secret/script +++ b/acceptance/cmd/workspace/put-secret/script @@ -1,7 +1,7 @@ scope="test-scope" trace $CLI secrets create-scope $scope -title "--string-value is sent verbatim and read back as-is\n" +title "--string-value is sent verbatim and read back as-is" trace $CLI secrets put-secret $scope string-key --string-value "test-value with-newlines " @@ -11,13 +11,13 @@ trace $CLI secrets get-secret $scope string-key # kept ASCII on purpose: Git Bash re-encodes non-UTF-8 argv bytes (\xfe becomes # \xc3\xbe), which would make the payload differ on Windows. A null byte is not # usable either, since bash drops it from "$(...)". -title "--bytes-value is base64-encoded by the CLI before it is sent\n" +title "--bytes-value is base64-encoded by the CLI before it is sent" bytes=$(printf '\x01\x02\x03') trace $CLI secrets put-secret $scope bytes-key --bytes-value "$bytes" trace $CLI secrets get-secret $scope bytes-key -title "the two value flags are mutually exclusive\n" +title "the two value flags are mutually exclusive" trace musterr $CLI secrets put-secret $scope k --string-value a --bytes-value b -title "what the CLI actually put on the wire\n" +title "what the CLI actually put on the wire" trace print_requests.py //api/2.0/secrets/put diff --git a/acceptance/cmd/workspace/uc-renames/output.txt b/acceptance/cmd/workspace/uc-renames/output.txt index 7c482c7a7e3..a075f958c02 100644 --- a/acceptance/cmd/workspace/uc-renames/output.txt +++ b/acceptance/cmd/workspace/uc-renames/output.txt @@ -1,5 +1,6 @@ === Create a catalog with a schema, and a volume, a registered model and a secret in it + >>> [CLI] catalogs create test_catalog_[UNIQUE_NAME] { "name": "test_catalog_[UNIQUE_NAME]" @@ -26,10 +27,12 @@ } === Creating a catalog under a taken name fails + >>> musterr [CLI] catalogs create test_catalog_[UNIQUE_NAME] Error: Catalog 'test_catalog_[UNIQUE_NAME]' already exists === Renaming the volume lowercases the new name + >>> [CLI] volumes update test_catalog_[UNIQUE_NAME].myschema.myvolume --new-name MyVolume2 { "full_name": "test_catalog_[UNIQUE_NAME].myschema.myvolume2", @@ -37,6 +40,7 @@ Error: Catalog 'test_catalog_[UNIQUE_NAME]' already exists } === Renaming the model keeps its full name in sync + >>> [CLI] registered-models update test_catalog_[UNIQUE_NAME].myschema.mymodel --new-name mymodel2 { "full_name": "test_catalog_[UNIQUE_NAME].myschema.mymodel2", @@ -44,6 +48,7 @@ Error: Catalog 'test_catalog_[UNIQUE_NAME]' already exists } === Renaming the schema moves the volume, the model and the secret with it + >>> [CLI] schemas update test_catalog_[UNIQUE_NAME].myschema --new-name myschema2 { "full_name": "test_catalog_[UNIQUE_NAME].myschema2" @@ -65,6 +70,7 @@ Error: Catalog 'test_catalog_[UNIQUE_NAME]' already exists } === Renaming the catalog moves everything in it + >>> [CLI] catalogs update test_catalog_[UNIQUE_NAME] --new-name test_catalog_renamed_[UNIQUE_NAME] { "name": "test_catalog_renamed_[UNIQUE_NAME]" diff --git a/acceptance/cmd/workspace/uc-renames/script b/acceptance/cmd/workspace/uc-renames/script index 68bbc7a575c..35e70bce5c3 100644 --- a/acceptance/cmd/workspace/uc-renames/script +++ b/acceptance/cmd/workspace/uc-renames/script @@ -41,14 +41,14 @@ trace $CLI volumes read "$RENAMED.myschema2.myvolume2" | jq '{full_name}' trace $CLI registered-models get "$RENAMED.myschema2.mymodel2" | jq '{full_name}' trace $CLI secrets-uc get-secret "$RENAMED.myschema2.mysecret" | jq '{full_name}' -title "Deleting the schema with force deletes everything in it\n" +title "Deleting the schema with force deletes everything in it" # Not traced: the "not found" messages differ between UC and the test server. $CLI schemas delete "$RENAMED.myschema2" --force echo "volume: $(exists $CLI volumes read "$RENAMED.myschema2.myvolume2")" echo "model: $(exists $CLI registered-models get "$RENAMED.myschema2.mymodel2")" echo "secret: $(exists $CLI secrets-uc get-secret "$RENAMED.myschema2.mysecret")" -title "Deleting the catalog with force deletes its schemas\n" +title "Deleting the catalog with force deletes its schemas" trace $CLI schemas create other "$RENAMED" | jq '{full_name}' $CLI catalogs delete "$RENAMED" --force echo "schema: $(exists $CLI schemas get "$RENAMED.other")" diff --git a/acceptance/experimental/aitools/skills/install-experimental-empty/output.txt b/acceptance/experimental/aitools/skills/install-experimental-empty/output.txt index c1a6249bf76..efdd0fb44ae 100644 --- a/acceptance/experimental/aitools/skills/install-experimental-empty/output.txt +++ b/acceptance/experimental/aitools/skills/install-experimental-empty/output.txt @@ -1,5 +1,6 @@ === --experimental against a manifest with no experimental skills logs a nudge + >>> [CLI] experimental aitools install --skills-only --global --experimental Command "install" is deprecated, use "databricks aitools install" instead. Flag --global has been deprecated, use --scope=global diff --git a/acceptance/experimental/aitools/skills/install-output-json-agent-error/output.txt b/acceptance/experimental/aitools/skills/install-output-json-agent-error/output.txt index 6860d375fa5..cf2f618437d 100644 --- a/acceptance/experimental/aitools/skills/install-output-json-agent-error/output.txt +++ b/acceptance/experimental/aitools/skills/install-output-json-agent-error/output.txt @@ -1,5 +1,6 @@ === install --output json records a per-agent error category + >>> [CLI] aitools install --skills-only --scope=project --agents=claude-code,cursor --output json { "scope": "project", diff --git a/acceptance/experimental/aitools/skills/install-output-json-agents-skipped/output.txt b/acceptance/experimental/aitools/skills/install-output-json-agents-skipped/output.txt index 01c511aaf17..2b5f8d403a8 100644 --- a/acceptance/experimental/aitools/skills/install-output-json-agents-skipped/output.txt +++ b/acceptance/experimental/aitools/skills/install-output-json-agents-skipped/output.txt @@ -1,5 +1,6 @@ === install --output json: every named agent skipped, each with its own category + >>> [CLI] aitools install --skills-only --scope=project --agents=cursor,codex --output json { "scope": "project", diff --git a/acceptance/experimental/aitools/skills/install-output-json-error/output.txt b/acceptance/experimental/aitools/skills/install-output-json-error/output.txt index db5a78e81e4..84ee93c8897 100644 --- a/acceptance/experimental/aitools/skills/install-output-json-error/output.txt +++ b/acceptance/experimental/aitools/skills/install-output-json-error/output.txt @@ -1,5 +1,6 @@ === install --output json reports a top-level error category and exits non-zero + >>> [CLI] aitools install --skills-only --scope=global --agents=claude-code --skills=nonexistent --output json { "scope": "global", diff --git a/acceptance/experimental/aitools/skills/install-output-json-experimental-error/output.txt b/acceptance/experimental/aitools/skills/install-output-json-experimental-error/output.txt index 513d2d5b5b3..53858b74eb4 100644 --- a/acceptance/experimental/aitools/skills/install-output-json-experimental-error/output.txt +++ b/acceptance/experimental/aitools/skills/install-output-json-experimental-error/output.txt @@ -1,5 +1,6 @@ === install --output json categorizes an experimental skill requested without --experimental + >>> [CLI] aitools install --skills-only --scope=global --agents=claude-code --skills=test-exp --output json { "scope": "global", diff --git a/acceptance/experimental/aitools/skills/install-output-json/output.txt b/acceptance/experimental/aitools/skills/install-output-json/output.txt index 5caa3daf14b..eb3b27dee51 100644 --- a/acceptance/experimental/aitools/skills/install-output-json/output.txt +++ b/acceptance/experimental/aitools/skills/install-output-json/output.txt @@ -1,5 +1,6 @@ === install --output json emits only the JSON document; progress is silenced + >>> [CLI] aitools install --skills-only --scope=global --agents=claude-code --output json { "scope": "global", diff --git a/acceptance/experimental/aitools/skills/install-specific/output.txt b/acceptance/experimental/aitools/skills/install-specific/output.txt index dc972dd17d3..03283c3f0d8 100644 --- a/acceptance/experimental/aitools/skills/install-specific/output.txt +++ b/acceptance/experimental/aitools/skills/install-specific/output.txt @@ -1,5 +1,6 @@ === install only one specific stable skill via --skills + >>> [CLI] experimental aitools install --skills-only --global --skills test-stable-a Command "install" is deprecated, use "databricks aitools install" instead. Flag --global has been deprecated, use --scope=global @@ -9,6 +10,7 @@ Fetching skills manifest... Installed 1 skill. === install a specific experimental skill + >>> [CLI] experimental aitools install --skills-only --global --skills test-exp --experimental Command "install" is deprecated, use "databricks aitools install" instead. Flag --global has been deprecated, use --scope=global @@ -18,6 +20,7 @@ Fetching skills manifest... Installed 1 skill. === asking for an experimental skill without --experimental flag errors out + >>> [CLI] experimental aitools install --skills-only --global --skills test-exp Command "install" is deprecated, use "databricks aitools install" instead. Flag --global has been deprecated, use --scope=global diff --git a/acceptance/experimental/aitools/skills/install/output.txt b/acceptance/experimental/aitools/skills/install/output.txt index bcd43f22227..82518161868 100644 --- a/acceptance/experimental/aitools/skills/install/output.txt +++ b/acceptance/experimental/aitools/skills/install/output.txt @@ -1,5 +1,6 @@ === stable-only install (no --experimental flag) installs 1 skill + >>> [CLI] experimental aitools install --skills-only --global Command "install" is deprecated, use "databricks aitools install" instead. Flag --global has been deprecated, use --scope=global @@ -9,6 +10,7 @@ Fetching skills manifest... Installed 1 skill. === re-run with --experimental installs the experimental one too + >>> [CLI] experimental aitools install --skills-only --global --experimental Command "install" is deprecated, use "databricks aitools install" instead. Flag --global has been deprecated, use --scope=global @@ -18,6 +20,7 @@ Fetching skills manifest... Installed 2 skills. === no-op re-run is idempotent (no new fetches, no errors) + >>> [CLI] experimental aitools install --skills-only --global --experimental Command "install" is deprecated, use "databricks aitools install" instead. Flag --global has been deprecated, use --scope=global diff --git a/acceptance/experimental/aitools/skills/path-dump/output.txt b/acceptance/experimental/aitools/skills/path-dump/output.txt index f5f7baeba90..b6efe765a8f 100644 --- a/acceptance/experimental/aitools/skills/path-dump/output.txt +++ b/acceptance/experimental/aitools/skills/path-dump/output.txt @@ -1,10 +1,12 @@ === dump skills to a directory + >>> [CLI] experimental aitools install --path dumped Command "install" is deprecated, use "databricks aitools install" instead. Using skills version test-ref Wrote 1 skill to dumped === the skill files were written + >>> find.py dumped.*SKILL.md --expect 1 dumped/test-stable/SKILL.md diff --git a/acceptance/experimental/aitools/skills/update-prune/output.txt b/acceptance/experimental/aitools/skills/update-prune/output.txt index 564957813a4..b777b02b3fa 100644 --- a/acceptance/experimental/aitools/skills/update-prune/output.txt +++ b/acceptance/experimental/aitools/skills/update-prune/output.txt @@ -1,5 +1,6 @@ === install two skills + >>> [CLI] experimental aitools install --skills-only --global Command "install" is deprecated, use "databricks aitools install" instead. Flag --global has been deprecated, use --scope=global @@ -9,6 +10,7 @@ Fetching skills manifest... Installed 2 skills. === update against a release where beta is gone: alpha updates, beta is pruned + >>> DATABRICKS_SKILLS_REF=v2-ref [CLI] experimental aitools update --scope global Command "update" is deprecated, use "databricks aitools update" instead. updated alpha v1.0.0 -> v2.0.0 diff --git a/acceptance/experimental/genie/ask-deprecated/output.txt b/acceptance/experimental/genie/ask-deprecated/output.txt index 475c7538824..9b2833f7a27 100644 --- a/acceptance/experimental/genie/ask-deprecated/output.txt +++ b/acceptance/experimental/genie/ask-deprecated/output.txt @@ -1,5 +1,6 @@ === the deprecated experimental alias still runs but prints a move notice + >>> [CLI] experimental genie ask What are total sales by franchise? Command "ask" is deprecated, use "databricks genie ask" instead; this experimental alias will be removed in a future release Alpha leads with 1,250 total sales. diff --git a/acceptance/experimental/open/output.txt b/acceptance/experimental/open/output.txt index f976b1b1794..09747c5baa0 100644 --- a/acceptance/experimental/open/output.txt +++ b/acceptance/experimental/open/output.txt @@ -1,17 +1,21 @@ === print URL for a job + >>> [CLI] experimental open --url jobs 123 [DATABRICKS_URL]/jobs/123?w=[WORKSPACE_ID] === print URL for a notebook + >>> [CLI] experimental open --url notebooks 12345 [DATABRICKS_URL]/?w=[WORKSPACE_ID]#notebook/12345 === unknown resource type + >>> [CLI] experimental open --url unknown 123 Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, cluster_policies, clusters, dashboards, database_catalogs, database_instances, experiments, feature_specs, features, genie_spaces, instance_pools, jobs, mcp_services, model_provider_services, model_services, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses === test auto-completion handler + >>> [CLI] __complete experimental open , alerts apps diff --git a/acceptance/genie/ask-endpoint-gone/output.txt b/acceptance/genie/ask-endpoint-gone/output.txt index b6e02a96ce7..4e12e61b911 100644 --- a/acceptance/genie/ask-endpoint-gone/output.txt +++ b/acceptance/genie/ask-endpoint-gone/output.txt @@ -1,4 +1,5 @@ === a removed endpoint tells the user to update the CLI + >>> [CLI] genie ask What are total sales by franchise? Error: the Genie API is not available on this workspace: No API found for 'POST /data-rooms/tools/onechat/responses'; the endpoint may have moved since this CLI release: update the Databricks CLI to the latest version (run 'databricks version --check') diff --git a/acceptance/genie/ask-protocol-drift/output.txt b/acceptance/genie/ask-protocol-drift/output.txt index 00e7f7f5689..9cb4613aced 100644 --- a/acceptance/genie/ask-protocol-drift/output.txt +++ b/acceptance/genie/ask-protocol-drift/output.txt @@ -1,11 +1,13 @@ === a drifted protocol with no renderable answer tells the user to update the CLI + >>> [CLI] genie ask What are total sales by franchise? Error: the stream ended without an answer (received 2 events); the API may have changed: update the Databricks CLI to the latest version (run 'databricks version --check'), or re-run with --raw to inspect the raw stream Exit code: 1 === json output also reports the drift + >>> [CLI] genie ask What are total sales by franchise? --output json { "status": "error", diff --git a/acceptance/genie/ask-request-drift/output.txt b/acceptance/genie/ask-request-drift/output.txt index 7be83fde191..6b8a35bb121 100644 --- a/acceptance/genie/ask-request-drift/output.txt +++ b/acceptance/genie/ask-request-drift/output.txt @@ -1,4 +1,5 @@ === a 500 with no message points at a possible request format change + >>> [CLI] genie ask What are total sales by franchise? Error: the Genie backend could not process the request (500 with no details); if this keeps happening, the request format may have changed since this CLI release: update the Databricks CLI to the latest version (run 'databricks version --check') diff --git a/acceptance/genie/ask/output.txt b/acceptance/genie/ask/output.txt index 397a096c2fb..81d61b37a8e 100644 --- a/acceptance/genie/ask/output.txt +++ b/acceptance/genie/ask/output.txt @@ -1,5 +1,6 @@ === ask renders the final answer and a chart + >>> [CLI] genie ask What are total sales by franchise? Alpha leads with 1,250 total sales. @@ -11,6 +12,7 @@ Alpha leads with 1,250 total sales. === ask with --include-sql also shows the executed SQL + >>> [CLI] genie ask What are total sales by franchise? --include-sql SQL executed (Total by Franchise): @@ -25,6 +27,7 @@ Alpha leads with 1,250 total sales. === ask with --output json emits structured output + >>> [CLI] genie ask What are total sales by franchise? --output json { "status": "completed", @@ -41,6 +44,7 @@ Alpha leads with 1,250 total sales. } === ask with --raw dumps raw SSE events + >>> [CLI] genie ask What are total sales by franchise? --raw {"type":"response.output_item.added","output_index":0,"item":{"type":"reasoning","id":"r1","status":"in_progress"}} {"type":"response.output_item.added","output_index":1,"item":{"type":"message","id":"t1","role":"assistant","status":"completed","content":[{"type":"output_text","text":"Looking at the sales data...","annotations":[]}],"metadata":{"ui_type":"THOUGHT","source_internal_ids":["m1"]}}} @@ -52,6 +56,7 @@ Alpha leads with 1,250 total sales. {"type":"response.completed","response":{"id":"resp_1","status":"completed","conversation_id":"conv_1"}} === incompatible flags are rejected + >>> [CLI] genie ask q --raw --output json Error: --raw cannot be used with --output json diff --git a/acceptance/pipelines/deploy/auto-approve/output.txt b/acceptance/pipelines/deploy/auto-approve/output.txt index ab4d3a151d7..6670fecf775 100644 --- a/acceptance/pipelines/deploy/auto-approve/output.txt +++ b/acceptance/pipelines/deploy/auto-approve/output.txt @@ -7,9 +7,11 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged View your pipeline foo here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === Remove resources from configuration to test auto-approve + >>> rm resources.yml === Try to redeploy without --auto-approve - should fail + >>> errcode [CLI] pipelines deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-auto-approve/default/files... @@ -27,6 +29,7 @@ Files: 0 uploaded, 1 deleted Exit code: 1 === Redeploy with --auto-approve - should succeed + >>> [CLI] pipelines deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-auto-approve/default/files... diff --git a/acceptance/pipelines/deploy/force-lock/output.txt b/acceptance/pipelines/deploy/force-lock/output.txt index e3922e9a536..c57d1704f09 100644 --- a/acceptance/pipelines/deploy/force-lock/output.txt +++ b/acceptance/pipelines/deploy/force-lock/output.txt @@ -1,8 +1,10 @@ === upload lock file + >>> [CLI] workspace import /Workspace/Users/[USERNAME]/.bundle/test-pipeline-force-lock/default/state/deploy.lock --format AUTO --file ./deploy.lock === test deployment without force-lock (should fail) + >>> errcode [CLI] pipelines deploy Error: Failed to acquire deployment lock: deploy lock acquired by user-with-lock@databricks.com at [TIMESTAMP] +0000 UTC. Another deployment may be in progress. Use --force-lock to override, but this may @@ -15,6 +17,7 @@ conflict with the other deployment if it is still active Exit code: 1 === test deployment with force-lock (should succeed) + >>> [CLI] pipelines deploy --force-lock Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-force-lock/default/files... Created pipelines.foo diff --git a/acceptance/pipelines/deploy/oss-spark-error/output.txt b/acceptance/pipelines/deploy/oss-spark-error/output.txt index 7be666bff44..ef801acc3ea 100644 --- a/acceptance/pipelines/deploy/oss-spark-error/output.txt +++ b/acceptance/pipelines/deploy/oss-spark-error/output.txt @@ -1,5 +1,6 @@ === Fail to deploy OSS Spark pipelines-formatted YAML file + >>> musterr [CLI] pipelines deploy Error: [TEST_TMP_DIR]/oss-pipeline.yml seems to be formatted for open-source Spark Declarative Pipelines. Pipelines CLI currently only supports Lakeflow Spark Declarative Pipelines development. diff --git a/acceptance/pipelines/deploy/var-flag/output.txt b/acceptance/pipelines/deploy/var-flag/output.txt index 01fa4e82c5e..4d9cda4cb24 100644 --- a/acceptance/pipelines/deploy/var-flag/output.txt +++ b/acceptance/pipelines/deploy/var-flag/output.txt @@ -1,5 +1,6 @@ === Test --var flag usage + >>> [CLI] pipelines deploy --var=catalog=custom_catalog --var=schema=custom_schema --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-var-flag/default/files... Created pipelines.foo @@ -8,6 +9,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged View your pipeline foo here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === Verify: Check that variables were substituted correctly + >>> [CLI] pipelines get [UUID] { "catalog": "custom_catalog", diff --git a/acceptance/pipelines/destroy/auto-approve/output.txt b/acceptance/pipelines/destroy/auto-approve/output.txt index d792a3a0942..ff0d319342a 100644 --- a/acceptance/pipelines/destroy/auto-approve/output.txt +++ b/acceptance/pipelines/destroy/auto-approve/output.txt @@ -1,5 +1,6 @@ === Test auto-approve pipelines destroy + >>> [CLI] pipelines deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy/default/files... Created pipelines.my_pipeline diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/output.txt b/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/output.txt index d2321f4fa70..471df0e732c 100644 --- a/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/output.txt +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/output.txt @@ -1,5 +1,6 @@ === Deploy a pipeline with cascade_on_destroy: true (state records true) + >>> [CLI] pipelines deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade-divergence/default/files... Created pipelines.my_pipeline @@ -23,7 +24,9 @@ View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSP } === Edit cascade_on_destroy true -> false in YAML WITHOUT deploying (config diverges from state) + === Destroy: banner and DELETE both reflect STATE (cascade=true), not the edited YAML + >>> [CLI] pipelines destroy --auto-approve The following resources will be deleted: delete resources.pipelines.my_pipeline diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt b/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt index 641d04b1af2..a9e1892612b 100644 --- a/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt @@ -1,5 +1,6 @@ === Deploy a pipeline configured with cascade_on_destroy: false + >>> [CLI] pipelines deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/files... Created pipelines.my_pipeline @@ -23,6 +24,7 @@ View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSP } === Change only cascade_on_destroy (false -> true): expect a state-only update, no pipeline update API call + >>> [CLI] pipelines deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/files... Updated pipelines.my_pipeline @@ -33,6 +35,7 @@ View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSP >>> print_requests.py //api/2.0/pipelines === Set cascade_on_destroy back to false and deploy so state reflects it before destroy + >>> [CLI] pipelines deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/files... Updated pipelines.my_pipeline @@ -43,6 +46,7 @@ View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSP >>> print_requests.py //api/2.0/pipelines === Unset cascade_on_destroy entirely: expect a state-only update that persists nil to state + >>> [CLI] pipelines deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/files... Updated pipelines.my_pipeline @@ -53,6 +57,7 @@ View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSP >>> print_requests.py //api/2.0/pipelines === Destroy: cascade_on_destroy is unset in state, so the delete uses the backend default (no cascade param) + >>> [CLI] pipelines destroy --auto-approve The following resources will be deleted: delete resources.pipelines.my_pipeline diff --git a/acceptance/pipelines/destroy/destroy-pipeline/output.txt b/acceptance/pipelines/destroy/destroy-pipeline/output.txt index a1456cf87e9..27b14d6c671 100644 --- a/acceptance/pipelines/destroy/destroy-pipeline/output.txt +++ b/acceptance/pipelines/destroy/destroy-pipeline/output.txt @@ -1,5 +1,6 @@ === Test basic pipelines destroy + >>> [CLI] pipelines deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy/default/files... Created pipelines.my_pipeline @@ -20,6 +21,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted === Assert pipeline is deleted + >>> errcode [CLI] pipelines get [UUID] Error: The specified pipeline [UUID] was not found. diff --git a/acceptance/pipelines/destroy/force-lock/output.txt b/acceptance/pipelines/destroy/force-lock/output.txt index 28c43d1bfd0..e4f477d6be4 100644 --- a/acceptance/pipelines/destroy/force-lock/output.txt +++ b/acceptance/pipelines/destroy/force-lock/output.txt @@ -7,9 +7,11 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged View your pipeline foo here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === upload lock file + >>> [CLI] workspace import /Workspace/Users/[USERNAME]/.bundle/test-pipeline-force-lock/default/state/deploy.lock --format AUTO --file ./deploy.lock === test deployment without force-lock (should fail) + >>> errcode [CLI] pipelines destroy --auto-approve Error: Failed to acquire deployment lock: deploy lock acquired by user-with-lock@databricks.com at [TIMESTAMP] +0000 UTC. Another deployment may be in progress. Use --force-lock to override, but this may @@ -22,6 +24,7 @@ conflict with the other deployment if it is still active Exit code: 1 === test deployment with force-lock (should succeed) + >>> [CLI] pipelines destroy --force-lock --auto-approve The following resources will be deleted: delete resources.pipelines.foo diff --git a/acceptance/pipelines/dry-run/dry-run-pipeline/output.txt b/acceptance/pipelines/dry-run/dry-run-pipeline/output.txt index 6470c25b206..7f6d018bc62 100644 --- a/acceptance/pipelines/dry-run/dry-run-pipeline/output.txt +++ b/acceptance/pipelines/dry-run/dry-run-pipeline/output.txt @@ -7,6 +7,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WORKSPACE_ID] === Dry running pipeline, should have validate_only set to true + >>> [CLI] pipelines dry-run Recommendation: This command runs the last deployed version of the code @@ -24,6 +25,7 @@ Update ID: [UUID][1] } === Dry running pipeline with KEY, expect same output as without KEY + >>> [CLI] pipelines dry-run my_pipeline Recommendation: This command runs the last deployed version of the code diff --git a/acceptance/pipelines/dry-run/no-wait/output.txt b/acceptance/pipelines/dry-run/no-wait/output.txt index e26d2ff1787..efef310efed 100644 --- a/acceptance/pipelines/dry-run/no-wait/output.txt +++ b/acceptance/pipelines/dry-run/no-wait/output.txt @@ -7,6 +7,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WORKSPACE_ID] === Dry running pipeline with --no-wait flag + >>> [CLI] pipelines dry-run --no-wait Recommendation: This command runs the last deployed version of the code diff --git a/acceptance/pipelines/dry-run/restart/output.txt b/acceptance/pipelines/dry-run/restart/output.txt index ef0456bd478..99a18977606 100644 --- a/acceptance/pipelines/dry-run/restart/output.txt +++ b/acceptance/pipelines/dry-run/restart/output.txt @@ -7,6 +7,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WORKSPACE_ID] === Dry running pipeline with --restart flag, should stop the current pipeline and start a new run + >>> [CLI] pipelines dry-run --restart Recommendation: This command runs the last deployed version of the code diff --git a/acceptance/pipelines/e2e/output.txt b/acceptance/pipelines/e2e/output.txt index 56ac7b696b3..33e90f819c8 100644 --- a/acceptance/pipelines/e2e/output.txt +++ b/acceptance/pipelines/e2e/output.txt @@ -1,6 +1,8 @@ === E2E Test: Complete pipeline lifecycle (init, deploy, run, stop, destroy) + === Initialize pipeline project + >>> [CLI] pipelines init --output-dir output Welcome to the template for Lakeflow Spark Declarative Pipelines! @@ -15,6 +17,7 @@ Note that [DATABRICKS_URL] is used for initialization Please refer to the README.md file for "getting started" instructions. === Deploy pipeline + >>> [CLI] pipelines deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lakeflow_project/dev/files... Created jobs.sample_job @@ -25,6 +28,7 @@ View your job sample_job here: [DATABRICKS_URL]/jobs/[SAMPLE_JOB_ID]?w=[WORKSPAC View your pipeline lakeflow_project_etl here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WORKSPACE_ID] === Run pipeline + >>> [CLI] pipelines run Recommendation: This command runs the last deployed version of the code @@ -40,6 +44,7 @@ Pipeline configurations for this update: • All tables are refreshed === Edit project by creating and running a new second pipeline + >>> [CLI] pipelines deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lakeflow_project/dev/files... Created pipelines.lakeflow_project_etl_2 @@ -50,6 +55,7 @@ View your pipeline lakeflow_project_etl here: [DATABRICKS_URL]/pipelines/[UUID][ View your pipeline lakeflow_project_etl_2 here: [DATABRICKS_URL]/pipelines/[UUID][2]?w=[WORKSPACE_ID] === Assert the second pipeline is created + >>> [CLI] pipelines get [UUID][2] { "creator_user_name": "[USERNAME]", @@ -91,6 +97,7 @@ Pipeline configurations for this update: • All tables are refreshed === Stop both pipelines before destroy + >>> [CLI] pipelines stop lakeflow_project_etl Stopping lakeflow_project_etl... lakeflow_project_etl has been stopped. @@ -100,6 +107,7 @@ Stopping lakeflow_project_etl_2... lakeflow_project_etl_2 has been stopped. === Destroy project + >>> [CLI] pipelines destroy --auto-approve The following resources will be deleted: delete resources.jobs.sample_job diff --git a/acceptance/pipelines/history/basic/script b/acceptance/pipelines/history/basic/script index 27a44231923..ff3b5a7b78f 100644 --- a/acceptance/pipelines/history/basic/script +++ b/acceptance/pipelines/history/basic/script @@ -1,10 +1,10 @@ add_repl 3fb8e5a1-0d2c-4a6b-9f1e-2c7d8e9f0a1b PIPELINE_ID -title "default history by pipeline id\n" +title "default history by pipeline id" trace $CLI pipelines history 3fb8e5a1-0d2c-4a6b-9f1e-2c7d8e9f0a1b -title "json output\n" +title "json output" trace $CLI pipelines history 3fb8e5a1-0d2c-4a6b-9f1e-2c7d8e9f0a1b --output json -title "wire request uses pipeline id\n" +title "wire request uses pipeline id" trace print_requests.py //api/2.0/pipelines/3fb8e5a1-0d2c-4a6b-9f1e-2c7d8e9f0a1b/updates --get diff --git a/acceptance/pipelines/init/error-cases/output.txt b/acceptance/pipelines/init/error-cases/output.txt index fc266593a14..a1b23dcde78 100644 --- a/acceptance/pipelines/init/error-cases/output.txt +++ b/acceptance/pipelines/init/error-cases/output.txt @@ -1,8 +1,10 @@ === Test with invalid project name (contains uppercase letters) + >>> musterr [CLI] pipelines init --config-file ./invalid_input.json --output-dir invalid-output Error: failed to load config from file ./invalid_input.json: invalid value for project_name: "InvalidProjectName". Name must consist of lower case letters, numbers, and underscores. === Test with non-existent config file + >>> musterr [CLI] pipelines init --config-file ./nonexistent.json --output-dir invalid-output-2 Error: failed to load config from file ./nonexistent.json: open ./nonexistent.json: no such file or directory diff --git a/acceptance/pipelines/init/python/output.txt b/acceptance/pipelines/init/python/output.txt index 708bdeb0b82..78e8d2910e7 100644 --- a/acceptance/pipelines/init/python/output.txt +++ b/acceptance/pipelines/init/python/output.txt @@ -1,5 +1,6 @@ === Test basic pipelines init with configuration file + >>> [CLI] pipelines init --config-file ./input.json --output-dir output Welcome to the template for Lakeflow Spark Declarative Pipelines! diff --git a/acceptance/pipelines/init/sql/output.txt b/acceptance/pipelines/init/sql/output.txt index 73f0e2f8469..a6b74849c8d 100644 --- a/acceptance/pipelines/init/sql/output.txt +++ b/acceptance/pipelines/init/sql/output.txt @@ -1,5 +1,6 @@ === Test pipelines init with SQL configuration + >>> [CLI] pipelines init --config-file ./input.json --output-dir output Welcome to the template for Lakeflow Spark Declarative Pipelines! diff --git a/acceptance/pipelines/open/open-after-deployment/output.txt b/acceptance/pipelines/open/open-after-deployment/output.txt index cf468e04dd6..9fbf2b94660 100644 --- a/acceptance/pipelines/open/open-after-deployment/output.txt +++ b/acceptance/pipelines/open/open-after-deployment/output.txt @@ -1,5 +1,6 @@ === not deployed yet + >>> [CLI] pipelines open Error: pipeline does not have a URL associated with it (has it been deployed?) @@ -13,12 +14,15 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged View your pipeline test-pipelines-open here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === Modify PATH so that real open is not run + === open after deployment. This will fail to open browser and complain, that's ok, we only want the message + >>> [CLI] pipelines open Opening browser at [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] Error: exec: "open": cannot run executable found relative to current directory === open with KEY, expect same output as opening without KEY + >>> [CLI] pipelines open test-pipelines-open Opening browser at [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] Error: exec: "open": cannot run executable found relative to current directory diff --git a/acceptance/pipelines/run/no-wait/output.txt b/acceptance/pipelines/run/no-wait/output.txt index 0378c4cfbcc..ea441339582 100644 --- a/acceptance/pipelines/run/no-wait/output.txt +++ b/acceptance/pipelines/run/no-wait/output.txt @@ -7,6 +7,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WORKSPACE_ID] === test --no-wait flag + >>> [CLI] pipelines run --no-wait Recommendation: This command runs the last deployed version of the code diff --git a/acceptance/pipelines/run/refresh-flags/output.txt b/acceptance/pipelines/run/refresh-flags/output.txt index 9887b899820..791a73fa999 100644 --- a/acceptance/pipelines/run/refresh-flags/output.txt +++ b/acceptance/pipelines/run/refresh-flags/output.txt @@ -7,6 +7,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WORKSPACE_ID] === Running pipeline with --refresh flag and specific tables + >>> [CLI] pipelines run --refresh table1,table2 Recommendation: This command runs the last deployed version of the code @@ -34,6 +35,7 @@ Pipeline configurations for this update: } === Running pipeline with --full-refresh-all flag + >>> [CLI] pipelines run --full-refresh-all Recommendation: This command runs the last deployed version of the code @@ -58,6 +60,7 @@ Pipeline configurations for this update: } === Running pipeline with --full-refresh flag and specific tables + >>> [CLI] pipelines run --full-refresh table1,table2 Recommendation: This command runs the last deployed version of the code @@ -85,6 +88,7 @@ Pipeline configurations for this update: } === Running pipeline with --full-refresh flag and --refresh flag + >>> [CLI] pipelines run --full-refresh table1,table2 --refresh table3,table4 Recommendation: This command runs the last deployed version of the code diff --git a/acceptance/pipelines/run/restart/output.txt b/acceptance/pipelines/run/restart/output.txt index f1098f0af08..57c1ac19357 100644 --- a/acceptance/pipelines/run/restart/output.txt +++ b/acceptance/pipelines/run/restart/output.txt @@ -7,6 +7,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WORKSPACE_ID] === Running pipeline with --restart flag + >>> [CLI] pipelines run --restart Recommendation: This command runs the last deployed version of the code diff --git a/acceptance/pipelines/run/run-info/output.txt b/acceptance/pipelines/run/run-info/output.txt index 755270d2611..7000c06b07f 100644 --- a/acceptance/pipelines/run/run-info/output.txt +++ b/acceptance/pipelines/run/run-info/output.txt @@ -7,6 +7,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === Run pipeline and display pipeline configuration summary and events durations + >>> [CLI] pipelines run Recommendation: This command runs the last deployed version of the code diff --git a/acceptance/pipelines/run/run-pipeline/output.txt b/acceptance/pipelines/run/run-pipeline/output.txt index f87465f0842..72679163527 100644 --- a/acceptance/pipelines/run/run-pipeline/output.txt +++ b/acceptance/pipelines/run/run-pipeline/output.txt @@ -1,5 +1,6 @@ === Run pipeline + >>> [CLI] pipelines run Error: resource not found or not yet deployed. Did you forget to run 'databricks bundle deploy'? diff --git a/acceptance/pipelines/show/basic/script b/acceptance/pipelines/show/basic/script index b63cfc73e23..3063d696c75 100644 --- a/acceptance/pipelines/show/basic/script +++ b/acceptance/pipelines/show/basic/script @@ -1,14 +1,14 @@ -title "table output\n" +title "table output" trace $CLI pipelines show main.default.people -title "json output\n" +title "json output" trace $CLI pipelines show main.default.people --output json -title "explicit limit does not show a hint\n" +title "explicit limit does not show a hint" trace $CLI pipelines show main.default.people --limit 9 -title "invalid table name\n" +title "invalid table name" musterr trace $CLI pipelines show people -title "wire SQL pins quoting and limits\n" +title "wire SQL pins quoting and limits" trace print_requests.py //api/2.0/sql/statements diff --git a/acceptance/pipelines/show/exact-limit/script b/acceptance/pipelines/show/exact-limit/script index 1b74a228537..cedb605e460 100644 --- a/acceptance/pipelines/show/exact-limit/script +++ b/acceptance/pipelines/show/exact-limit/script @@ -1,5 +1,5 @@ -title "default limit does not show a hint when exactly 10 rows are available\n" +title "default limit does not show a hint when exactly 10 rows are available" trace $CLI pipelines show main.default.people -title "wire SQL fetches one extra row\n" +title "wire SQL fetches one extra row" trace print_requests.py //api/2.0/sql/statements diff --git a/acceptance/pipelines/show/help/output.txt b/acceptance/pipelines/show/help/output.txt index 67a94f1bdb1..a2cc2ddc948 100644 --- a/acceptance/pipelines/show/help/output.txt +++ b/acceptance/pipelines/show/help/output.txt @@ -1,5 +1,6 @@ === help + >>> [CLI] pipelines show --help Preview a table's columns and sample rows on a SQL warehouse. diff --git a/acceptance/pipelines/show/limit-hint/script b/acceptance/pipelines/show/limit-hint/script index 7f066a8fe9c..604ed80f4fb 100644 --- a/acceptance/pipelines/show/limit-hint/script +++ b/acceptance/pipelines/show/limit-hint/script @@ -1,5 +1,5 @@ -title "default limit shows a hint when more rows are available\n" +title "default limit shows a hint when more rows are available" trace $CLI pipelines show main.default.people -title "wire SQL fetches one extra row\n" +title "wire SQL fetches one extra row" trace print_requests.py //api/2.0/sql/statements diff --git a/acceptance/pipelines/show/rendering/script b/acceptance/pipelines/show/rendering/script index 5ae4498b154..96d6c76a6c2 100644 --- a/acceptance/pipelines/show/rendering/script +++ b/acceptance/pipelines/show/rendering/script @@ -1,5 +1,5 @@ -title "table output: long note truncated, ragged row blank cell\n" +title "table output: long note truncated, ragged row blank cell" trace $CLI pipelines show main.default.people -title "json output: column order preserved, ragged row padded with null\n" +title "json output: column order preserved, ragged row padded with null" trace $CLI pipelines show main.default.people --output json diff --git a/acceptance/pipelines/stop/output.txt b/acceptance/pipelines/stop/output.txt index 1f14be93164..e010544aabc 100644 --- a/acceptance/pipelines/stop/output.txt +++ b/acceptance/pipelines/stop/output.txt @@ -21,6 +21,7 @@ Pipeline configurations for this update: • All tables are refreshed === Stopping pipeline + >>> [CLI] pipelines stop Stopping my_pipeline... my_pipeline has been stopped. @@ -30,6 +31,7 @@ my_pipeline has been stopped. } === Stopping pipeline with KEY, expect same output as without KEY + >>> [CLI] pipelines stop my_pipeline Stopping my_pipeline... my_pipeline has been stopped. diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index 99a5cd9912f..0e7d4dfb200 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -41,10 +41,12 @@ Global Flags: --var strings set values for variables defined in project config. Example: --var="foo=bar" === The command is experimental and off by default + >>> musterr [CLI] pipelines test --pipeline-id 1234 Error: pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it === pytest arguments must be passed with --pytest-arg + >>> musterr [CLI] pipelines test --pipeline-id 1234 -k orders Error: -k is a pytest argument: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow' @@ -52,6 +54,7 @@ Error: -k is a pytest argument: pass pytest arguments with --pytest-arg, for exa Error: the -- separator is not supported: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow' === Invalid --pytest-arg values fail before any request + >>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-x Error: unsupported pytest argument "-x": only -k and -m are supported @@ -59,9 +62,11 @@ Error: unsupported pytest argument "-x": only -k and -m are supported Error: --pytest-arg requires a non-empty value === The pipeline ID must be a pipeline ID + >>> musterr [CLI] pipelines test --pipeline-id 1234 Error: --pipeline-id "1234" is not a pipeline ID === JSON output is not supported + >>> musterr [CLI] pipelines test --pipeline-id 1234 -o json Error: --output json is not supported by pipelines test diff --git a/acceptance/pipelines/test/immutable/output.txt b/acceptance/pipelines/test/immutable/output.txt index 02f1cbaef42..e18544efea3 100644 --- a/acceptance/pipelines/test/immutable/output.txt +++ b/acceptance/pipelines/test/immutable/output.txt @@ -7,6 +7,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WORKSPACE_ID] === Local test paths are not supported with workspace.sync_mode immutable + >>> musterr [CLI] pipelines test my_pipeline tests/test_orders.py::test_backfill Recommendation: This command runs the last deployed version of the code @@ -15,6 +16,7 @@ If you've made local changes, run 'databricks pipelines deploy' first to ensure Error: cannot resolve test selection "tests/test_orders.py::test_backfill": local test paths are not supported with workspace.sync_mode immutable; pass an absolute workspace path === Absolute workspace paths and running all tests still work + >>> MSYS_NO_PATHCONV=1 [CLI] pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill Recommendation: This command runs the last deployed version of the code diff --git a/acceptance/pipelines/test/pipeline-id/output.txt b/acceptance/pipelines/test/pipeline-id/output.txt index 9b40c5ca1fa..4664393739b 100644 --- a/acceptance/pipelines/test/pipeline-id/output.txt +++ b/acceptance/pipelines/test/pipeline-id/output.txt @@ -1,5 +1,6 @@ === Relative test selections resolve against the pipeline root_path + >>> MSYS_NO_PATHCONV=1 [CLI] pipelines test --pipeline-id [UUID][0] tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/other/tests Pipeline: [UUID][0] Update: [UUID][1] @@ -20,6 +21,7 @@ Update: [UUID][1] } === Without a root_path, test selections must be workspace paths + >>> musterr [CLI] pipelines test --pipeline-id [UUID][2] tests Error: cannot resolve test selection "tests": the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests @@ -42,6 +44,7 @@ Update: [UUID][3] } === Outside a project, --pipeline-id is required + >>> musterr [CLI] pipelines test tests Error: Unable to locate the bundle root: databricks.yml not found. diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt index 9a9e80db6e6..47d75a6bb74 100644 --- a/acceptance/pipelines/test/selection/output.txt +++ b/acceptance/pipelines/test/selection/output.txt @@ -7,6 +7,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WORKSPACE_ID] === Run all tests of the only pipeline + >>> [CLI] pipelines test Recommendation: This command runs the last deployed version of the code @@ -25,6 +26,7 @@ Update: [UUID][1] } === Select tests by relative path and test ID, with pytest arguments in both --pytest-arg forms + >>> MSYS_NO_PATHCONV=1 [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Users/someone@example.com/tests/test_other.py --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg -m --pytest-arg smoke,fast Recommendation: This command runs the last deployed version of the code @@ -56,6 +58,7 @@ Update: [UUID][2] } === Relative paths resolve from the working directory + >>> [CLI] pipelines test my_pipeline test_orders.py::test_valid_order Recommendation: This command runs the last deployed version of the code @@ -79,5 +82,6 @@ Update: [UUID][3] } === Test selections require the pipeline KEY first + >>> musterr [CLI] pipelines test tests/test_orders.py Error: "tests/test_orders.py" is not a pipeline in this bundle (pipelines: my_pipeline). Usage: databricks pipelines test my_pipeline tests/test_orders.py diff --git a/acceptance/pipelines/test/submit-error/output.txt b/acceptance/pipelines/test/submit-error/output.txt index fc3de068652..32a27c892d3 100644 --- a/acceptance/pipelines/test/submit-error/output.txt +++ b/acceptance/pipelines/test/submit-error/output.txt @@ -1,5 +1,6 @@ === A submit error from the service is shown as is, and no update ID is printed + >>> musterr [CLI] pipelines test --pipeline-id [UUID][0] Error: An active update '[UUID][1]' already exists for pipeline '[UUID][0]'. diff --git a/acceptance/script.prepare b/acceptance/script.prepare index e74b77d388d..506e03ce2be 100644 --- a/acceptance/script.prepare +++ b/acceptance/script.prepare @@ -70,7 +70,10 @@ git-repo-init() { title() { local label="$1" - printf "\n=== %b" "$label" + # Print a trailing newline so the header is self-contained: it no longer + # relies on a following `trace` (which prepends one) to terminate the line, + # and output that is not a `trace` no longer gets mashed onto the header. + printf "\n=== %b\n" "$label" } withdir() { diff --git a/acceptance/selftest/contains/output.txt b/acceptance/selftest/contains/output.txt index 181578f94fa..528c95a2396 100644 --- a/acceptance/selftest/contains/output.txt +++ b/acceptance/selftest/contains/output.txt @@ -1,5 +1,6 @@ === This should complain about not_found not found + >>> python3 ./success.py Hello world contains error: 'not_found' not found in the output. @@ -7,6 +8,7 @@ contains error: 'not_found' not found in the output. Exit code: 1 === This should complain about not_found not found + >>> musterr python3 ./failure.py Failed script contains error: 'not_found' not found in the output. @@ -14,14 +16,17 @@ contains error: 'not_found' not found in the output. Exit code: 1 === This should not complain + >>> python3 ./success.py Hello world === This should not complain + >>> musterr python3 ./failure.py Failed script === This should complain about Hello present in output + >>> python3 ./success.py Hello world contains error: 'Hello' was not expected: 'Hello world' @@ -29,5 +34,6 @@ contains error: 'Hello' was not expected: 'Hello world' Exit code: 1 === This should not complain + >>> python3 ./success.py Hello world diff --git a/acceptance/selftest/record_cloud/pipeline-crud/output.txt b/acceptance/selftest/record_cloud/pipeline-crud/output.txt index 97ac1421a4b..cef3d2f9eef 100644 --- a/acceptance/selftest/record_cloud/pipeline-crud/output.txt +++ b/acceptance/selftest/record_cloud/pipeline-crud/output.txt @@ -23,6 +23,7 @@ } === Get the pipeline + >>> [CLI] pipelines get [UUID] "test-pipeline-1" @@ -33,6 +34,7 @@ } === Update the pipeline + >>> [CLI] pipelines update [UUID] --json @pipeline2.json >>> print_requests @@ -53,6 +55,7 @@ } === Verify the update + >>> [CLI] pipelines get [UUID] "test-pipeline-2" @@ -63,6 +66,7 @@ } === Delete the pipeline + >>> [CLI] pipelines delete [UUID] >>> print_requests @@ -72,6 +76,7 @@ } === Verify the deletion + >>> [CLI] pipelines get [UUID] Error: The specified pipeline [UUID] was not found. diff --git a/acceptance/selftest/record_cloud/pipeline-crud/script b/acceptance/selftest/record_cloud/pipeline-crud/script index 4e6ada990d8..cb85f035483 100644 --- a/acceptance/selftest/record_cloud/pipeline-crud/script +++ b/acceptance/selftest/record_cloud/pipeline-crud/script @@ -6,7 +6,6 @@ print_requests() { # Verify that the entire crud lifecycle of a pipeline works. title "Create a pipeline" export pipeline_id=$($CLI pipelines create --json @pipeline1.json | jq -r .pipeline_id) -echo "" trace print_requests diff --git a/acceptance/selftest/record_cloud/workspace-file-io/output.txt b/acceptance/selftest/record_cloud/workspace-file-io/output.txt index 7f515ba1511..1a41e2c3a79 100644 --- a/acceptance/selftest/record_cloud/workspace-file-io/output.txt +++ b/acceptance/selftest/record_cloud/workspace-file-io/output.txt @@ -1,5 +1,6 @@ === create a folder + >>> [CLI] workspace mkdirs /Users/[USERNAME]/[UNIQUE_NAME] >>> print_requests @@ -20,6 +21,7 @@ } === upload a file + >>> [CLI] workspace import /Users/[USERNAME]/[UNIQUE_NAME]/hello.txt --format AUTO --file ./hello.txt >>> print_requests @@ -34,6 +36,7 @@ } === list the folder + >>> [CLI] workspace list /Users/[USERNAME]/[UNIQUE_NAME] --output json { "path": "/Users/[USERNAME]/[UNIQUE_NAME]/hello.txt", @@ -50,6 +53,7 @@ } === stat the file + >>> [CLI] workspace get-status /Users/[USERNAME]/[UNIQUE_NAME]/hello.txt { "path": "/Users/[USERNAME]/[UNIQUE_NAME]/hello.txt", @@ -66,6 +70,7 @@ } === download the file + >>> [CLI] workspace export /Users/[USERNAME]/[UNIQUE_NAME]/hello.txt --format AUTO --file ./hello2.txt >>> cat hello2.txt @@ -82,6 +87,7 @@ hello, world } === delete the file + >>> [CLI] workspace delete /Users/[USERNAME]/[UNIQUE_NAME]/hello.txt >>> print_requests @@ -94,6 +100,7 @@ hello, world } === stat the file again + >>> [CLI] workspace get-status /Users/[USERNAME]/[UNIQUE_NAME]/hello.txt Error: Path (/Users/[USERNAME]/[UNIQUE_NAME]/hello.txt) doesn't exist. @@ -109,10 +116,12 @@ Exit code: 1 } === list the folder + >>> [CLI] workspace list /Users/[USERNAME]/[UNIQUE_NAME] ID Type Language Path === delete the folder + >>> [CLI] workspace delete /Users/[USERNAME]/[UNIQUE_NAME] >>> print_requests @@ -132,6 +141,7 @@ ID Type Language Path } === list the folder again + >>> [CLI] workspace list /Users/[USERNAME]/[UNIQUE_NAME] Error: Path (/Users/[USERNAME]/[UNIQUE_NAME]) doesn't exist. diff --git a/acceptance/selftest/selection/script b/acceptance/selftest/selection/script index 122f4f4c8bb..4d7fc146d47 100644 --- a/acceptance/selftest/selection/script +++ b/acceptance/selftest/selection/script @@ -6,31 +6,31 @@ selection() { # Each argument is a changed file: "M:" modified, "A:" added, "R100:old:new" moved. -title "Changing a test\n" +title "Changing a test" trace selection M:acceptance/selftest/basic/script -title "Adding a test\n" +title "Adding a test" trace selection A:acceptance/selftest/basic/script -title "A regenerated golden scores below a changed fixture\n" +title "A regenerated golden scores below a changed fixture" trace selection M:acceptance/selftest/basic/output.txt M:acceptance/selftest/diff/script -title "Moving a test scores lowest\n" +title "Moving a test scores lowest" trace selection R100:acceptance/selftest/old/script:acceptance/selftest/basic/script M:acceptance/selftest/diff/script -title "Touching an invariant config runs every invariant test, for that config only\n" +title "Touching an invariant config runs every invariant test, for that config only" trace selection M:acceptance/bundle/invariant/configs/job.yml.tmpl -title "Adding one invariant config and touching another: only the new variant is new\n" +title "Adding one invariant config and touching another: only the new variant is new" trace selection A:acceptance/bundle/invariant/configs/pipeline.yml.tmpl M:acceptance/bundle/invariant/configs/job.yml.tmpl -title "A config an invariant test excludes does not select that test\n" +title "A config an invariant test excludes does not select that test" # continue_293 pins an old CLI that has no Vector Search support, so it excludes this config; # selecting it would spend the limit on a test whose every variant is skipped. trace selection M:acceptance/bundle/invariant/configs/vector_search_index.yml.tmpl -title "Renaming a fixture out of a test selects the test that lost it\n" +title "Renaming a fixture out of a test selects the test that lost it" trace selection R100:acceptance/selftest/basic/output.txt:docs/example.txt -title "The limit keeps the highest scoring tests\n" +title "The limit keeps the highest scoring tests" trace selection -limit 3 A:acceptance/selftest/basic/script M:acceptance/selftest/diff/script M:acceptance/bundle/invariant/configs/job.yml.tmpl diff --git a/acceptance/selftest/selection_exclusions/script b/acceptance/selftest/selection_exclusions/script index 6dd48e78021..b82dea4c365 100644 --- a/acceptance/selftest/selection_exclusions/script +++ b/acceptance/selftest/selection_exclusions/script @@ -19,7 +19,7 @@ selection() { "$SELECTION" -root acceptance } -title "A new template excluded from migration selects only its no_drift variant\n" +title "A new template excluded from migration selects only its no_drift variant" cp ../fixtures/new.yml.tmpl acceptance/bundle/invariant/configs/ cp ../fixtures/excluded.toml acceptance/bundle/invariant/migrate/test.toml cp ../fixtures/new_matrix.toml acceptance/bundle/invariant/no_drift/out.test.toml @@ -27,7 +27,7 @@ git add . trace cat acceptance/bundle/invariant/migrate/test.toml trace selection -title "Changing the migration script also selects its existing variants\n" +title "Changing the migration script also selects its existing variants" cp ../fixtures/changed_script.txt acceptance/bundle/invariant/migrate/script trace selection @@ -36,7 +36,7 @@ cp ../fixtures/script.txt acceptance/bundle/invariant/migrate/script git commit -qm 'Add template and exclusion' git update-ref refs/remotes/origin/main HEAD -title "Removing the exclusion selects migration, including the re-enabled variant\n" +title "Removing the exclusion selects migration, including the re-enabled variant" cp ../fixtures/base.toml acceptance/bundle/invariant/migrate/test.toml cp ../fixtures/new_matrix.toml acceptance/bundle/invariant/migrate/out.test.toml trace cat acceptance/bundle/invariant/migrate/test.toml diff --git a/acceptance/selftest/uuid/script b/acceptance/selftest/uuid/script index 4a123e5f405..86641008783 100644 --- a/acceptance/selftest/uuid/script +++ b/acceptance/selftest/uuid/script @@ -1,4 +1,4 @@ -title "Distinct UUIDs are numbered\n" +title "Distinct UUIDs are numbered" # Distinct UUIDs get a stable [N] suffix in order of first appearance, so a golden # pins which value appears where; a repeated UUID keeps its number. echo "first: 11111111-1111-1111-1111-111111111111" diff --git a/acceptance/ssh/agent-launchers/script b/acceptance/ssh/agent-launchers/script index 84c72a3d375..b8761c2bd42 100644 --- a/acceptance/ssh/agent-launchers/script +++ b/acceptance/ssh/agent-launchers/script @@ -20,22 +20,22 @@ $CLI secrets put-secret ssh-scope client-public-key --string-value "$(cat client # delay makes the server exit on its own once it is set up. SERVER_HOME="$PWD/server-home" mkdir -p "$SERVER_HOME" -title "Serverless server\n" +title "Serverless server" env -i HOME="$SERVER_HOME" PATH=/usr/bin:/bin DATABRICKS_HOST="$DATABRICKS_HOST" DATABRICKS_TOKEN="$DATABRICKS_TOKEN" \ $CLI ssh server --serverless --cluster=test-cluster --session-id=test-session --secret-scope-name=ssh-scope \ --authorized-key-secret-name=client-public-key --shutdown-delay=1s 2>LOG.server # One variable per line; no value here contains a space. -title "SetEnv appends the launcher directory to PATH\n" +title "SetEnv appends the launcher directory to PATH" grep '^SetEnv ' server-home/.ssh-tunnel/sshd_config | tr ' ' '\n' -title "Each supported agent has a launcher that delegates to the agent shim\n" +title "Each supported agent has a launcher that delegates to the agent shim" trace ls server-home/.agent-shim/bin trace cat server-home/.agent-shim/bin/claude # A non-interactive session, as "ssh connect -- " or an IDE's remote exec opens: # ssh runs the command through the login shell without a PTY. -title "A non-interactive session finds the launchers on PATH\n" +title "A non-interactive session finds the launchers on PATH" ssh -F /dev/null -i client_key -l "$(id -un)" \ -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \ -o ProxyCommand="$SSHD -i -f $SERVER_HOME/.ssh-tunnel/sshd_config -o StrictModes=no" \ @@ -44,17 +44,17 @@ ssh -F /dev/null -i client_key -l "$(id -un)" \ # Dedicated clusters get the same launchers and PATH entry as serverless. rm -rf "$SERVER_HOME" mkdir -p "$SERVER_HOME" -title "Dedicated server\n" +title "Dedicated server" env -i HOME="$SERVER_HOME" PATH=/usr/bin:/bin DATABRICKS_HOST="$DATABRICKS_HOST" DATABRICKS_TOKEN="$DATABRICKS_TOKEN" \ $CLI ssh server --cluster=test-cluster --session-id=test-session --secret-scope-name=ssh-scope \ --authorized-key-secret-name=client-public-key --shutdown-delay=1s 2>LOG.dedicated-server -title "SetEnv appends the launcher directory to PATH\n" +title "SetEnv appends the launcher directory to PATH" grep '^SetEnv ' server-home/.ssh-tunnel/sshd_config | tr ' ' '\n' | grep '^PATH=' trace ls server-home/.agent-shim/bin -title "A non-interactive session finds the launchers on PATH\n" +title "A non-interactive session finds the launchers on PATH" ssh -F /dev/null -i client_key -l "$(id -un)" \ -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \ -o ProxyCommand="$SSHD -i -f $SERVER_HOME/.ssh-tunnel/sshd_config -o StrictModes=no" \ diff --git a/acceptance/ssh/bulk-transfer/script b/acceptance/ssh/bulk-transfer/script index 7272d7d56f1..9a93e87d3b8 100644 --- a/acceptance/ssh/bulk-transfer/script +++ b/acceptance/ssh/bulk-transfer/script @@ -37,11 +37,11 @@ else CONNECT_FLAGS="--releases-dir=$PWD/releases --known-hosts-dir=known-hosts" fi -title "Download $PAYLOAD_BYTES bytes over the tunnel\n" +title "Download $PAYLOAD_BYTES bytes over the tunnel" errcode $CLI ssh connect $CONNECT_FLAGS -- "$payload_cmd" >received.bin 2>LOG.download echo "received $(wc -c sent.bin errcode $CLI ssh connect $CONNECT_FLAGS -- "wc -c" remote-count.txt 2>LOG.upload # The remote count is empty when the session ended before `wc` could report; say 0 rather than diff --git a/acceptance/ssh/connect-serverless-cpu/script b/acceptance/ssh/connect-serverless-cpu/script index 14c787e58ea..40e5fa6c2be 100644 --- a/acceptance/ssh/connect-serverless-cpu/script +++ b/acceptance/ssh/connect-serverless-cpu/script @@ -1,6 +1,6 @@ if [ -n "${CLOUD_ENV:-}" ]; then # Cloud: real serverless CPU runs the command; dump the run on failure. - title "CLI ssh connect (serverless CPU)\n" + title "CLI ssh connect (serverless CPU)" errcode $CLI ssh connect --releases-dir=$CLI_RELEASES_DIR -- "echo 'Connection successful'" >out.stdout.txt 2>LOG.stderr if ! grep -q "Connection successful" out.stdout.txt; then run_id=$(cat LOG.stderr | grep -o "Job submitted successfully with run ID: [0-9]*" | grep -o "[0-9]*$") @@ -21,7 +21,7 @@ else fake_release releases/databricks_cli_linux_arm64.zip CLI_RELEASES_DIR="$PWD/releases" - title "CLI ssh connect (local sshd)\n" + title "CLI ssh connect (local sshd)" errcode $CLI ssh connect --releases-dir=$CLI_RELEASES_DIR --known-hosts-dir=known-hosts -- "echo 'Connection successful'" >out.stdout.txt 2>LOG.stderr trace print_requests.py //api/2.2/jobs/runs/submit fi diff --git a/acceptance/ssh/connect-serverless-feature-disabled/script b/acceptance/ssh/connect-serverless-feature-disabled/script index 67901dc76ef..ec2c8f393d3 100644 --- a/acceptance/ssh/connect-serverless-feature-disabled/script +++ b/acceptance/ssh/connect-serverless-feature-disabled/script @@ -11,12 +11,12 @@ mkdir -p releases fake_release releases/databricks_cli_linux_amd64.zip fake_release releases/databricks_cli_linux_arm64.zip -title "CLI ssh connect\n" +title "CLI ssh connect" musterr $CLI ssh connect --name feature-disabled --releases-dir="$PWD/releases" -- "echo 'Connection successful'" >out.connect.txt 2>&1 add_repl "$(grep -o 'run ID: [0-9]*' out.connect.txt | grep -o '[0-9]*$')" RUN_ID -title "Reported error category\n" +title "Reported error category" trace jq -r 'select(.path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.ssh_tunnel_event.error_category' "$OUT_REQUESTS" -title "Each release archive is uploaded once, and the submitted run is cancelled\n" +title "Each release archive is uploaded once, and the submitted run is cancelled" trace print_requests.py --del-field raw_body //api/2.0/workspace-files/import-file //api/2.2/jobs/runs/cancel diff --git a/acceptance/ssh/connect-serverless-port-forwarding-disabled/script b/acceptance/ssh/connect-serverless-port-forwarding-disabled/script index 2dbc12b6ac0..78b69b07cf3 100644 --- a/acceptance/ssh/connect-serverless-port-forwarding-disabled/script +++ b/acceptance/ssh/connect-serverless-port-forwarding-disabled/script @@ -6,10 +6,10 @@ trace $CLI workspace-settings-v2 patch-public-workspace-setting port_fwd_serverl # Drop the setup request so the check below only sees what ssh connect sent. print_requests.py //api/2.1/settings >/dev/null -title "CLI ssh connect\n" +title "CLI ssh connect" trace musterr $CLI ssh connect --name port-forwarding-disabled -- "echo 'Connection successful'" -title "Reported error category\n" +title "Reported error category" trace jq -r 'select(.path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.ssh_tunnel_event.error_category' "$OUT_REQUESTS" # Expect no requests other than reads: the connection stops at the settings check. diff --git a/acceptance/ssh/reconnect-reused-name/script b/acceptance/ssh/reconnect-reused-name/script index a0ac016f93f..7a325b10a7e 100644 --- a/acceptance/ssh/reconnect-reused-name/script +++ b/acceptance/ssh/reconnect-reused-name/script @@ -27,10 +27,10 @@ echo "reused-name ssh-ed25519 $STALE_KEY" >known-hosts/reused-name # The remote command's output is the assertion (see out.connect-*.txt); the tunnel's own # progress and teardown messages go to stderr, where the order of the last few lines # depends on which side closes the websocket first. -title "CLI ssh connect over a stale host key for the connection name\n" +title "CLI ssh connect over a stale host key for the connection name" errcode $CLI ssh connect --name reused-name --releases-dir=$CLI_RELEASES_DIR --known-hosts-dir=known-hosts -- "echo 'Connection successful'" >out.connect-1.txt 2>LOG.connect-1 -title "The stale key is replaced by the one the workspace published\n" +title "The stale key is replaced by the one the workspace published" if grep -q "$STALE_KEY" known-hosts/reused-name; then echo "stale key still pinned" else @@ -38,5 +38,5 @@ else fi trace grep -c "^reused-name ssh-" known-hosts/reused-name -title "Reconnecting to the running server keeps working\n" +title "Reconnecting to the running server keeps working" errcode $CLI ssh connect --name reused-name --releases-dir=$CLI_RELEASES_DIR --known-hosts-dir=known-hosts -- "echo 'Connection successful'" >out.connect-2.txt 2>LOG.connect-2 diff --git a/acceptance/ssh/setup/output.txt b/acceptance/ssh/setup/output.txt index c07eb2cc138..2801eda5bb3 100644 --- a/acceptance/ssh/setup/output.txt +++ b/acceptance/ssh/setup/output.txt @@ -9,10 +9,12 @@ ssh connect --proxy --cluster=[TEST_DEFAULT_CLUSTER_ID] --auto-start-cluster=tru ssh connect --proxy --cluster=[TEST_DEFAULT_CLUSTER_ID] --auto-start-cluster=true --shutdown-delay=10m0s --keep-detached-processes --max-clients=10 --server-timeout=24h0m0s === Rejects a server that would refuse every connection + >>> [CLI] ssh setup --name=broken --cluster=[TEST_DEFAULT_CLUSTER_ID] --max-clients=0 Error: --max-clients must be at least 1, got 0 === Rejects a shutdown delay the server can never reach + >>> [CLI] ssh setup --name=broken --cluster=[TEST_DEFAULT_CLUSTER_ID] --shutdown-delay=48h --server-timeout=24h Error: --shutdown-delay (48h0m0s) cannot be longer than --server-timeout (24h0m0s) diff --git a/acceptance/ssh/setup/script b/acceptance/ssh/setup/script index a04c038ff3d..dc7a65d5d36 100644 --- a/acceptance/ssh/setup/script +++ b/acceptance/ssh/setup/script @@ -7,19 +7,19 @@ $CLI ssh setup --name=my-cluster --cluster=$TEST_DEFAULT_CLUSTER_ID --max-client # The ProxyCommand is the invocation that submits the SSH server job, so --max-clients and # --server-timeout have to appear there or `ssh my-cluster` silently starts a server with the # defaults. -title "ProxyCommand written by setup --max-clients=25 --server-timeout=48h\n" +title "ProxyCommand written by setup --max-clients=25 --server-timeout=48h" sed -n 's/.*\(ssh connect --proxy.*\)/\1/p' "$HOME/.databricks/ssh-tunnel-configs/my-cluster" # A shutdown delay longer than the default 24h lifetime worked before --server-timeout existed, # and `ssh setup` persisted it into the ProxyCommand. OpenSSH runs that ProxyCommand verbatim, so # the delay has to keep raising the lifetime rather than becoming an error the user cannot fix. -title "A shutdown delay beyond the default lifetime raises it, no --server-timeout needed\n" +title "A shutdown delay beyond the default lifetime raises it, no --server-timeout needed" $CLI ssh setup --name=long-delay --cluster=$TEST_DEFAULT_CLUSTER_ID --shutdown-delay=48h &>LOG.long-delay sed -n 's/.*\(ssh connect --proxy.*\)/\1/p' "$HOME/.databricks/ssh-tunnel-configs/long-delay" # Holding the job run open for detached processes is fixed at submission too, so a host # configured through setup can only ask for it here. -title "ProxyCommand written by setup --keep-detached-processes\n" +title "ProxyCommand written by setup --keep-detached-processes" $CLI ssh setup --name=keep-detached --cluster=$TEST_DEFAULT_CLUSTER_ID --keep-detached-processes &>LOG.keep-detached sed -n 's/.*\(ssh connect --proxy.*\)/\1/p' "$HOME/.databricks/ssh-tunnel-configs/keep-detached" @@ -29,5 +29,5 @@ musterr trace $CLI ssh setup --name=broken --cluster=$TEST_DEFAULT_CLUSTER_ID -- title "Rejects a shutdown delay the server can never reach" musterr trace $CLI ssh setup --name=broken --cluster=$TEST_DEFAULT_CLUSTER_ID --shutdown-delay=48h --server-timeout=24h -title "No host config is written for the rejected setups\n" +title "No host config is written for the rejected setups" find.py 'ssh-tunnel-configs' --expect 3 diff --git a/acceptance/workspace/repos/create_with_provider/output.txt b/acceptance/workspace/repos/create_with_provider/output.txt index 07d9eac9623..68987f4fe4f 100644 --- a/acceptance/workspace/repos/create_with_provider/output.txt +++ b/acceptance/workspace/repos/create_with_provider/output.txt @@ -2,6 +2,7 @@ >>> [CLI] repos create https://github.com/databricks/databricks-empty-ide-project.git gitHub --path /Repos/me@databricks.com/test-repo === Get by id should work + >>> [CLI] repos get [REPO_ID] -o json { "branch": "main", @@ -13,6 +14,7 @@ } === Get by path should work + >>> [CLI] repos get /Repos/me@databricks.com/test-repo -o json { "branch": "main", @@ -24,6 +26,7 @@ } === Delete by id should work + >>> [CLI] repos delete [REPO_ID] >>> print_requests.py --get //repos //workspace/get-status diff --git a/acceptance/workspace/repos/git_cli_folder/output.txt b/acceptance/workspace/repos/git_cli_folder/output.txt index f27bd344833..d0f71380985 100644 --- a/acceptance/workspace/repos/git_cli_folder/output.txt +++ b/acceptance/workspace/repos/git_cli_folder/output.txt @@ -3,6 +3,7 @@ [REPO_ID] === Get by path resolves a Git CLI folder + >>> [CLI] repos get /Workspace/Users/me@databricks.com/test-repo -o json { "branch": "main", @@ -14,9 +15,11 @@ } === Update by path resolves a Git CLI folder + >>> [CLI] repos update /Workspace/Users/me@databricks.com/test-repo --branch update-by-path === Delete by path resolves a Git CLI folder + >>> [CLI] repos delete /Workspace/Users/me@databricks.com/test-repo >>> print_requests.py --get //repos //workspace/get-status