diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.json b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.json index 0471f4d7f85..ae4b80b519d 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.json +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.json @@ -17,13 +17,11 @@ }, "remote_state": { "assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]", - "dashboard_id": "(redacted)", "drift_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_drift_metrics", "monitor_version": 0, "output_schema_name": "main.qm_test_[UNIQUE_NAME]", "profile_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_profile_metrics", "snapshot": {}, - "status": "MONITOR_STATUS_ACTIVE", "table_name": "main.qm_test_[UNIQUE_NAME].test_table" }, "changes": { diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script index 5ea603ee6fe..e752b9417ab 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script @@ -27,7 +27,7 @@ trace errcode $CLI bundle plan &> out.plan.txt # Untraced: with &> the trace header lands in the file, leaving it unfilterable. Capture the raw # plan aside and stamp it out into the golden instead. errcode $CLI bundle plan -o json &> .databricks/tmp.plan.json -nostamp < .databricks/tmp.plan.json > out.plan.json +plan_monitor_fields < .databricks/tmp.plan.json | nostamp > out.plan.json rm "$OUT_REQUESTS" trace errcode $CLI bundle deploy &> out.deploy.txt diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.json b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.json index 7d37b225a2a..17c543171b6 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.json +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.json @@ -17,13 +17,11 @@ }, "remote_state": { "assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]", - "dashboard_id": "(redacted)", "drift_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_drift_metrics", "monitor_version": 0, "output_schema_name": "main.qm_test_[UNIQUE_NAME]", "profile_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_profile_metrics", "snapshot": {}, - "status": "MONITOR_STATUS_ACTIVE", "table_name": "main.qm_test_[UNIQUE_NAME].test_table" }, "changes": { diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script index fbd62ade8df..7aecb5320b2 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script @@ -22,7 +22,7 @@ trace $CLI bundle plan | contains.py "1 unchanged" update_file.py databricks.yml "output_schema_name: main.qm_test_${UNIQUE_NAME}" "output_schema_name: main.qm_test_${UNIQUE_NAME}_2" trace $CLI bundle plan | contains.py "1 to change" -trace $CLI bundle plan -o json | nostamp > out.plan.json +trace $CLI bundle plan -o json | plan_monitor_fields | nostamp > out.plan.json rm "$OUT_REQUESTS" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.get.json b/acceptance/bundle/resources/quality_monitors/change_table_name/out.get.json index f6e2bce85de..95596a82c24 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/out.get.json +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.get.json @@ -1,11 +1,9 @@ { "assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]", - "dashboard_id": "(redacted)", "drift_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_2_drift_metrics", "monitor_version": 0, "output_schema_name": "main.qm_test_[UNIQUE_NAME]", "profile_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_2_profile_metrics", "snapshot": {}, - "status": "MONITOR_STATUS_ACTIVE", "table_name": "main.qm_test_[UNIQUE_NAME].test_table_2" } diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.json b/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.json index 850061e1c60..a040211f65d 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.json +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.json @@ -17,13 +17,11 @@ }, "remote_state": { "assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]", - "dashboard_id": "(redacted)", "drift_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_drift_metrics", "monitor_version": 0, "output_schema_name": "main.qm_test_[UNIQUE_NAME]", "profile_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_profile_metrics", "snapshot": {}, - "status": "MONITOR_STATUS_ACTIVE", "table_name": "main.qm_test_[UNIQUE_NAME].test_table" }, "changes": { diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/script b/acceptance/bundle/resources/quality_monitors/change_table_name/script index ca49568b38a..839046985b1 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/script @@ -27,11 +27,11 @@ trace errcode $CLI bundle plan &> out.plan.txt # Untraced: with &> the trace header lands in the file, leaving it unfilterable. Capture the raw # plan aside and stamp it out into the golden instead. errcode $CLI bundle plan -o json &> .databricks/tmp.plan.json -nostamp < .databricks/tmp.plan.json > out.plan.json +plan_monitor_fields < .databricks/tmp.plan.json | nostamp > out.plan.json rm "$OUT_REQUESTS" trace errcode $CLI bundle deploy &> out.deploy.txt trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json trace errcode $CLI bundle plan &> out.plan_after_deploy.txt -trace errcode $CLI quality-monitors get ${TABLE_NAME}_2 2> /dev/null > out.get.json +trace errcode $CLI quality-monitors get ${TABLE_NAME}_2 2> /dev/null | monitor_fields > out.get.json diff --git a/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.json b/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.json index 310bb1515b3..2ffff85807a 100644 --- a/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.json +++ b/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.json @@ -8,13 +8,11 @@ "action": "skip", "remote_state": { "assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]", - "dashboard_id": "(redacted)", "drift_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_drift_metrics", "monitor_version": 0, "output_schema_name": "main.qm_test_[UNIQUE_NAME]", "profile_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_profile_metrics", "snapshot": {}, - "status": "MONITOR_STATUS_ACTIVE", "table_name": "main.qm_test_[UNIQUE_NAME].test_table" }, "changes": { diff --git a/acceptance/bundle/resources/quality_monitors/create/script b/acceptance/bundle/resources/quality_monitors/create/script index b45b8226f4e..3f23747f720 100644 --- a/acceptance/bundle/resources/quality_monitors/create/script +++ b/acceptance/bundle/resources/quality_monitors/create/script @@ -26,4 +26,4 @@ trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^ # store state to ensure we have table_name there print_state.py | grep name > out.state.txt -trace $CLI bundle plan -o json | nostamp > out.plan_noop.json +trace $CLI bundle plan -o json | plan_monitor_fields | nostamp > out.plan_noop.json diff --git a/acceptance/bundle/resources/quality_monitors/script.prepare b/acceptance/bundle/resources/quality_monitors/script.prepare new file mode 100644 index 00000000000..68e84f0a502 --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/script.prepare @@ -0,0 +1,16 @@ +# monitor_fields is the allow-list of quality monitor fields the acceptance tests +# assert on. Anything the backend returns outside this list is dropped, so a new +# field in the API response does not break the golden. status and dashboard_id +# are deliberately left out: right after create, the backend can still report +# MONITOR_STATUS_PENDING with no dashboard_id yet. +MONITOR_FIELDS='{assets_dir, drift_metrics_table_name, monitor_version, output_schema_name, profile_metrics_table_name, snapshot, table_name}' + +monitor_fields() { + jq "$MONITOR_FIELDS" +} + +# plan_monitor_fields applies the same allow-list to remote_state in a +# `bundle plan -o json` output. +plan_monitor_fields() { + jq "(.plan[] | select(.remote_state)).remote_state |= $MONITOR_FIELDS" +} diff --git a/acceptance/bundle/resources/quality_monitors/test.toml b/acceptance/bundle/resources/quality_monitors/test.toml index 4e383d30499..9a99197e216 100644 --- a/acceptance/bundle/resources/quality_monitors/test.toml +++ b/acceptance/bundle/resources/quality_monitors/test.toml @@ -6,7 +6,3 @@ RecordRequests = true [[Server]] Pattern = "POST /api/2.0/sql/statements/" Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' - -[[Repls]] -Old = '"dashboard_id":\s*"[0-9a-f]+",' -New = '"dashboard_id": "(redacted)",'