Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,11 @@
},
"remote_state": {
"assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]",
"dashboard_id": "(redacted)",
"drift_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_drift_metrics",
"monitor_version": 0,
"output_schema_name": "main.qm_test_[UNIQUE_NAME]",
"profile_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_profile_metrics",
"snapshot": {},
"status": "MONITOR_STATUS_ACTIVE",
"table_name": "main.qm_test_[UNIQUE_NAME].test_table"
},
"changes": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ trace errcode $CLI bundle plan &> out.plan.txt
# Untraced: with &> the trace header lands in the file, leaving it unfilterable. Capture the raw
# plan aside and stamp it out into the golden instead.
errcode $CLI bundle plan -o json &> .databricks/tmp.plan.json
nostamp < .databricks/tmp.plan.json > out.plan.json
plan_monitor_fields < .databricks/tmp.plan.json | nostamp > out.plan.json

rm "$OUT_REQUESTS"
trace errcode $CLI bundle deploy &> out.deploy.txt
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,11 @@
},
"remote_state": {
"assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]",
"dashboard_id": "(redacted)",
"drift_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_drift_metrics",
"monitor_version": 0,
"output_schema_name": "main.qm_test_[UNIQUE_NAME]",
"profile_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_profile_metrics",
"snapshot": {},
"status": "MONITOR_STATUS_ACTIVE",
"table_name": "main.qm_test_[UNIQUE_NAME].test_table"
},
"changes": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ trace $CLI bundle plan | contains.py "1 unchanged"
update_file.py databricks.yml "output_schema_name: main.qm_test_${UNIQUE_NAME}" "output_schema_name: main.qm_test_${UNIQUE_NAME}_2"

trace $CLI bundle plan | contains.py "1 to change"
trace $CLI bundle plan -o json | nostamp > out.plan.json
trace $CLI bundle plan -o json | plan_monitor_fields | nostamp > out.plan.json

rm "$OUT_REQUESTS"
trace $CLI bundle deploy
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
{
"assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]",
"dashboard_id": "(redacted)",
"drift_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_2_drift_metrics",
"monitor_version": 0,
"output_schema_name": "main.qm_test_[UNIQUE_NAME]",
"profile_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_2_profile_metrics",
"snapshot": {},
"status": "MONITOR_STATUS_ACTIVE",
"table_name": "main.qm_test_[UNIQUE_NAME].test_table_2"
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,11 @@
},
"remote_state": {
"assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]",
"dashboard_id": "(redacted)",
"drift_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_drift_metrics",
"monitor_version": 0,
"output_schema_name": "main.qm_test_[UNIQUE_NAME]",
"profile_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_profile_metrics",
"snapshot": {},
"status": "MONITOR_STATUS_ACTIVE",
"table_name": "main.qm_test_[UNIQUE_NAME].test_table"
},
"changes": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@ trace errcode $CLI bundle plan &> out.plan.txt
# Untraced: with &> the trace header lands in the file, leaving it unfilterable. Capture the raw
# plan aside and stamp it out into the golden instead.
errcode $CLI bundle plan -o json &> .databricks/tmp.plan.json
nostamp < .databricks/tmp.plan.json > out.plan.json
plan_monitor_fields < .databricks/tmp.plan.json | nostamp > out.plan.json

rm "$OUT_REQUESTS"
trace errcode $CLI bundle deploy &> out.deploy.txt
trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json
trace errcode $CLI bundle plan &> out.plan_after_deploy.txt

trace errcode $CLI quality-monitors get ${TABLE_NAME}_2 2> /dev/null > out.get.json
trace errcode $CLI quality-monitors get ${TABLE_NAME}_2 2> /dev/null | monitor_fields > out.get.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,11 @@
"action": "skip",
"remote_state": {
"assets_dir": "/Workspace/Users/[USERNAME]/monitor_assets_[UNIQUE_NAME]",
"dashboard_id": "(redacted)",
"drift_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_drift_metrics",
"monitor_version": 0,
"output_schema_name": "main.qm_test_[UNIQUE_NAME]",
"profile_metrics_table_name": "main.qm_test_[UNIQUE_NAME].test_table_profile_metrics",
"snapshot": {},
"status": "MONITOR_STATUS_ACTIVE",
"table_name": "main.qm_test_[UNIQUE_NAME].test_table"
},
"changes": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,4 @@ trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^
# store state to ensure we have table_name there
print_state.py | grep name > out.state.txt

trace $CLI bundle plan -o json | nostamp > out.plan_noop.json
trace $CLI bundle plan -o json | plan_monitor_fields | nostamp > out.plan_noop.json
16 changes: 16 additions & 0 deletions acceptance/bundle/resources/quality_monitors/script.prepare
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# monitor_fields is the allow-list of quality monitor fields the acceptance tests
# assert on. Anything the backend returns outside this list is dropped, so a new
# field in the API response does not break the golden. status and dashboard_id
# are deliberately left out: right after create, the backend can still report
# MONITOR_STATUS_PENDING with no dashboard_id yet.
MONITOR_FIELDS='{assets_dir, drift_metrics_table_name, monitor_version, output_schema_name, profile_metrics_table_name, snapshot, table_name}'

monitor_fields() {
jq "$MONITOR_FIELDS"
}

# plan_monitor_fields applies the same allow-list to remote_state in a
# `bundle plan -o json` output.
plan_monitor_fields() {
jq "(.plan[] | select(.remote_state)).remote_state |= $MONITOR_FIELDS"
}
4 changes: 0 additions & 4 deletions acceptance/bundle/resources/quality_monitors/test.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,3 @@ RecordRequests = true
[[Server]]
Pattern = "POST /api/2.0/sql/statements/"
Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}'

[[Repls]]
Old = '"dashboard_id":\s*"[0-9a-f]+",'
New = '"dashboard_id": "(redacted)",'
Loading