From 34bc6035c6f8a8cfcdadc75ea84905e2d55d073e Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Wed, 11 Nov 2026 18:37:13 +0000 Subject: [PATCH 1/2] Address the feature spec resource review follow-ups Review feedback on #7364 left for after the merge: the test server models the fixture SQL statements instead of per-test stubs, the fixture helpers are shared, the FeatureSpec approval prompts name every consequence, and capture_uc_dependencies shares one three-part-name helper. Co-authored-by: Isaac --- .../bin/create_feature_spec_fixtures.py | 51 +++++------------ acceptance/bin/create_table.py | 31 +++-------- acceptance/bin/util.py | 18 ++++++ .../deployment/bind/feature_spec/output.txt | 5 +- .../deployment/bind/feature_spec/script | 17 +----- .../deployment/bind/feature_spec/test.toml | 5 -- .../configs/feature_spec.yml.tmpl-init.sh | 5 +- acceptance/bundle/invariant/test.toml | 5 -- .../feature_specs-support/databricks.yml | 3 - .../feature_specs-support/out.test.toml | 2 +- .../python/feature_specs-support/output.txt | 8 --- .../python/feature_specs-support/test.toml | 2 +- .../resources/feature_specs/basic/output.txt | 9 +-- .../resources/feature_specs/order/output.txt | 2 +- .../feature_specs/remote-delete/output.txt | 3 +- .../feature_specs/remote-delete/script | 2 +- .../bundle/resources/feature_specs/test.toml | 9 +-- .../feature_specs/uppercase-name/output.txt | 7 ++- .../resources/features/lifecycle/test.toml | 5 -- .../resources/grants/feature_specs/output.txt | 7 ++- .../resources/grants/feature_specs/script | 18 +----- .../resources/grants/feature_specs/test.toml | 5 -- .../feature_specs => }/script.prepare | 15 +++-- .../capture_uc_dependencies.go | 49 +++++++---------- bundle/direct/dresources/all_test.go | 29 +++++++--- bundle/direct/dresources/feature_spec.go | 15 ++--- bundle/phases/messages.go | 7 ++- libs/testserver/feature_spec.go | 6 -- libs/testserver/handlers.go | 11 ++++ libs/testserver/statements_test.go | 55 +++++++++++++++++++ libs/testserver/testsql/testsql.go | 21 ++++++- libs/testserver/testsql/testsql_test.go | 13 +++++ 32 files changed, 232 insertions(+), 208 deletions(-) rename acceptance/bundle/{resources/feature_specs => }/script.prepare (51%) diff --git a/acceptance/bin/create_feature_spec_fixtures.py b/acceptance/bin/create_feature_spec_fixtures.py index 78fe9ed9b05..4ca06250c45 100755 --- a/acceptance/bin/create_feature_spec_fixtures.py +++ b/acceptance/bin/create_feature_spec_fixtures.py @@ -6,44 +6,21 @@ Usage: create_feature_spec_fixtures.py CATALOG.SCHEMA """ -import json -import os -import subprocess import sys -import time - -schema_full_name = sys.argv[1] -cli = os.environ.get("CLI", "databricks") -warehouse_id = os.environ["TEST_DEFAULT_WAREHOUSE_ID"] - -statements = [ - f"CREATE TABLE {schema_full_name}.user_features " - "(user_id INT NOT NULL, age INT, country STRING, CONSTRAINT user_features_pk PRIMARY KEY (user_id))", - f"CREATE TABLE {schema_full_name}.tenant_user_features " - "(tenant_id INT NOT NULL, user_id INT NOT NULL, age INT, country STRING, " - "CONSTRAINT tenant_user_features_pk PRIMARY KEY (tenant_id, user_id))", - f"CREATE FUNCTION {schema_full_name}.score(x INT) RETURNS DOUBLE LANGUAGE PYTHON AS $$ return float(x) $$", - f"CREATE FUNCTION {schema_full_name}.country_score(x INT, y STRING) RETURNS DOUBLE LANGUAGE PYTHON " - "AS $$ return float(x) + len(y) $$", -] +from util import run_statements -def call_api(*args): - result = subprocess.run([cli, "api", *args], capture_output=True, text=True) - if result.returncode != 0: - print(f"databricks api {' '.join(args)} failed: {result.stderr}", file=sys.stderr) - sys.exit(1) - return json.loads(result.stdout) - +schema_full_name = sys.argv[1] -for statement in statements: - payload = json.dumps({"warehouse_id": warehouse_id, "statement": statement, "wait_timeout": "50s"}) - response = call_api("post", "/api/2.0/sql/statements/", "--json", payload) - # A warehouse that is still starting outlasts wait_timeout. - deadline = time.monotonic() + 600 - while response["status"]["state"] in ("PENDING", "RUNNING") and time.monotonic() < deadline: - time.sleep(5) - response = call_api("get", f"/api/2.0/sql/statements/{response['statement_id']}") - if response["status"]["state"] != "SUCCEEDED": - print(f"Statement {statement!r} ended in {response['status']}", file=sys.stderr) - sys.exit(1) +run_statements( + [ + f"CREATE TABLE {schema_full_name}.user_features " + "(user_id INT NOT NULL, age INT, country STRING, CONSTRAINT user_features_pk PRIMARY KEY (user_id))", + f"CREATE TABLE {schema_full_name}.tenant_user_features " + "(tenant_id INT NOT NULL, user_id INT NOT NULL, age INT, country STRING, " + "CONSTRAINT tenant_user_features_pk PRIMARY KEY (tenant_id, user_id))", + f"CREATE FUNCTION {schema_full_name}.score(x INT) RETURNS DOUBLE LANGUAGE PYTHON AS $$ return float(x) $$", + f"CREATE FUNCTION {schema_full_name}.country_score(x INT, y STRING) RETURNS DOUBLE LANGUAGE PYTHON " + "AS $$ return float(x) + len(y) $$", + ] +) diff --git a/acceptance/bin/create_table.py b/acceptance/bin/create_table.py index 1fb028dda18..fb04c4cfd8b 100755 --- a/acceptance/bin/create_table.py +++ b/acceptance/bin/create_table.py @@ -5,6 +5,8 @@ import sys import time +from util import run_statements + table_name = sys.argv[1] # Extract catalog.schema from table_name parts = table_name.split(".") @@ -24,36 +26,19 @@ def run_cli(*args): return result -def execute_sql(warehouse_id, sql): - """Execute SQL using the API endpoint.""" - payload = json.dumps({"warehouse_id": warehouse_id, "statement": sql, "wait_timeout": "30s"}) - return run_cli("api", "post", "/api/2.0/sql/statements/", "--json", payload) - - -# Get warehouse ID from environment variable -warehouse_id = os.environ["TEST_DEFAULT_WAREHOUSE_ID"] - # Create a simple table -sql = f"CREATE TABLE IF NOT EXISTS {table_name} (id INT, value STRING, timestamp TIMESTAMP) USING DELTA" - -result = execute_sql(warehouse_id, sql) -if result.returncode != 0: - print(f"Failed to create table: {result.stderr}", file=sys.stderr) - sys.exit(1) - +run_statements([f"CREATE TABLE IF NOT EXISTS {table_name} (id INT, value STRING, timestamp TIMESTAMP) USING DELTA"]) print(f"Created table {table_name}") # Insert some sample data so the monitor has something to analyze -insert_sql = f"""INSERT INTO {table_name} VALUES +run_statements( + [ + f"""INSERT INTO {table_name} VALUES (1, 'test1', current_timestamp()), (2, 'test2', current_timestamp()), (3, 'test3', current_timestamp())""" - -result = execute_sql(warehouse_id, insert_sql) -if result.returncode != 0: - print(f"Failed to insert data: {result.stderr}", file=sys.stderr) - sys.exit(1) - + ] +) print(f"Inserted sample data into {table_name}") # Wait for table to be visible in Unity Catalog diff --git a/acceptance/bin/util.py b/acceptance/bin/util.py index 3ee8d65bc90..d7045670433 100644 --- a/acceptance/bin/util.py +++ b/acceptance/bin/util.py @@ -1,7 +1,9 @@ import json +import os import shlex import subprocess import sys +import time VERBOSE = False @@ -31,3 +33,19 @@ def run(cmd): if result.returncode != 0: raise RunError(f"{cmd} failed with code {result.returncode}") return result + + +def run_statements(statements): + """Runs SQL statements on the test warehouse one after another, waiting for each to succeed.""" + cli = os.environ.get("CLI", "databricks") + warehouse_id = os.environ["TEST_DEFAULT_WAREHOUSE_ID"] + for statement in statements: + payload = json.dumps({"warehouse_id": warehouse_id, "statement": statement, "wait_timeout": "50s"}) + response = run_json([cli, "api", "post", "/api/2.0/sql/statements/", "--json", payload]) + # A warehouse that is still starting outlasts wait_timeout. + deadline = time.monotonic() + 600 + while response["status"]["state"] in ("PENDING", "RUNNING") and time.monotonic() < deadline: + time.sleep(5) + response = run_json([cli, "api", "get", f"/api/2.0/sql/statements/{response['statement_id']}"]) + if response["status"]["state"] != "SUCCEEDED": + raise RunError(f"Statement {statement!r} ended in {response['status']}") diff --git a/acceptance/bundle/deployment/bind/feature_spec/output.txt b/acceptance/bundle/deployment/bind/feature_spec/output.txt index a983fb36577..a859db162c0 100644 --- a/acceptance/bundle/deployment/bind/feature_spec/output.txt +++ b/acceptance/bundle/deployment/bind/feature_spec/output.txt @@ -8,7 +8,7 @@ === Create the feature spec outside the bundle ->>> [CLI] api post /api/2.0/feature-engineering/feature-specs --json { +>>> [CLI] feature-engineering create-feature-spec --json { "name": "main.fspec_[UNIQUE_NAME].boundspec", "features": [{"feature_lookup": {"table": "main.fspec_[UNIQUE_NAME].user_features", "lookup_keys": ["user_id"], "feature_names": ["age"]}}] } @@ -51,5 +51,4 @@ Resources: Name: main.fspec_[UNIQUE_NAME].boundspec URL: (not deployed) ->>> [CLI] api delete /api/2.0/feature-engineering/feature-specs/main.fspec_[UNIQUE_NAME].boundspec -{} +>>> [CLI] feature-engineering delete-feature-spec main.fspec_[UNIQUE_NAME].boundspec diff --git a/acceptance/bundle/deployment/bind/feature_spec/script b/acceptance/bundle/deployment/bind/feature_spec/script index cbe4e43b07e..608c7fc8edc 100644 --- a/acceptance/bundle/deployment/bind/feature_spec/script +++ b/acceptance/bundle/deployment/bind/feature_spec/script @@ -1,18 +1,7 @@ -# Creating a feature spec resolves its lookup tables in Unity Catalog, so the test creates them in -# a schema of its own. Dropping the schema also drops the feature spec. -export SCHEMA_FULL_NAME="main.fspec_$UNIQUE_NAME" -trace $CLI schemas create "fspec_$UNIQUE_NAME" main -o json | jq '{full_name}' -trace create_feature_spec_fixtures.py "$SCHEMA_FULL_NAME" - -cleanup() { - $CLI schemas delete "$SCHEMA_FULL_NAME" --force >> LOG.cleanup 2>&1 || true -} -trap cleanup EXIT - -envsubst < databricks.yml.tmpl > databricks.yml +feature_spec_setup title "Create the feature spec outside the bundle" -MSYS_NO_PATHCONV=1 trace $CLI api post /api/2.0/feature-engineering/feature-specs --json "{ +trace $CLI feature-engineering create-feature-spec --json "{ \"name\": \"$SCHEMA_FULL_NAME.boundspec\", \"features\": [{\"feature_lookup\": {\"table\": \"$SCHEMA_FULL_NAME.user_features\", \"lookup_keys\": [\"user_id\"], \"feature_names\": [\"age\"]}}] }" | jq '{name}' @@ -26,4 +15,4 @@ trace $CLI bundle plan trace $CLI bundle deployment unbind spec1 trace $CLI bundle summary -MSYS_NO_PATHCONV=1 trace $CLI api delete /api/2.0/feature-engineering/feature-specs/$SCHEMA_FULL_NAME.boundspec +trace $CLI feature-engineering delete-feature-spec $SCHEMA_FULL_NAME.boundspec diff --git a/acceptance/bundle/deployment/bind/feature_spec/test.toml b/acceptance/bundle/deployment/bind/feature_spec/test.toml index e41de801cf3..66cb91f3551 100644 --- a/acceptance/bundle/deployment/bind/feature_spec/test.toml +++ b/acceptance/bundle/deployment/bind/feature_spec/test.toml @@ -2,11 +2,6 @@ Ignore = [ ".databricks", ] -# Fake SQL endpoint so create_feature_spec_fixtures.py works against the local test server too. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' - # Azure and GCP hosts carry the workspace ID, so their summary URLs have no ?w=. [[Repls]] Old = '\?[ow]=\[WORKSPACE_ID\]' diff --git a/acceptance/bundle/invariant/configs/feature_spec.yml.tmpl-init.sh b/acceptance/bundle/invariant/configs/feature_spec.yml.tmpl-init.sh index 91158f1d7ce..986943a23b5 100755 --- a/acceptance/bundle/invariant/configs/feature_spec.yml.tmpl-init.sh +++ b/acceptance/bundle/invariant/configs/feature_spec.yml.tmpl-init.sh @@ -2,7 +2,4 @@ # Creating a feature spec resolves its lookup table in Unity Catalog, and the invariant harness # provisions no tables of its own. -echo "Creating temporary schema and feature table: main.fspec_inv_$UNIQUE_NAME" - -$CLI schemas create fspec_inv_$UNIQUE_NAME main -o json | jq '{full_name}' -create_feature_spec_fixtures.py main.fspec_inv_$UNIQUE_NAME +feature_spec_fixtures "fspec_inv_$UNIQUE_NAME" diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 04a46bf44d4..c18ebb8b5fe 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -130,11 +130,6 @@ EnvMatrixExclude.no_feature_dms = ["DMS=true", "INPUT_CONFIG=feature.yml.tmpl"] EnvMatrixExclude.no_feature_spec_dms = ["DMS=true", "INPUT_CONFIG=feature_spec.yml.tmpl"] # Same for the "postgres_data_apis." prefix. EnvMatrixExclude.no_postgres_data_api_dms = ["DMS=true", "INPUT_CONFIG=postgres_data_api.yml.tmpl"] -# Fake SQL endpoint for local tests - -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" diff --git a/acceptance/bundle/python/feature_specs-support/databricks.yml b/acceptance/bundle/python/feature_specs-support/databricks.yml index c4ded89a6eb..201ed896e2a 100644 --- a/acceptance/bundle/python/feature_specs-support/databricks.yml +++ b/acceptance/bundle/python/feature_specs-support/databricks.yml @@ -18,6 +18,3 @@ resources: table: main.default.user_features lookup_keys: [user_id] feature_names: [age] - grants: - - principal: account users - privileges: [EXECUTE] diff --git a/acceptance/bundle/python/feature_specs-support/out.test.toml b/acceptance/bundle/python/feature_specs-support/out.test.toml index ea77b758035..aeddf3e619e 100644 --- a/acceptance/bundle/python/feature_specs-support/out.test.toml +++ b/acceptance/bundle/python/feature_specs-support/out.test.toml @@ -1,3 +1,3 @@ -Cloud = true +Cloud = false EnvMatrix.DMS = ["", "true"] EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/python/feature_specs-support/output.txt b/acceptance/bundle/python/feature_specs-support/output.txt index db824f305b3..8982e0071c1 100644 --- a/acceptance/bundle/python/feature_specs-support/output.txt +++ b/acceptance/bundle/python/feature_specs-support/output.txt @@ -27,14 +27,6 @@ } } ], - "grants": [ - { - "principal": "account users", - "privileges": [ - "EXECUTE" - ] - } - ], "name": "main.default.my_feature_spec_1_updated" }, "my_feature_spec_2": { diff --git a/acceptance/bundle/python/feature_specs-support/test.toml b/acceptance/bundle/python/feature_specs-support/test.toml index 6aee90e30c0..d0aada9733d 100644 --- a/acceptance/bundle/python/feature_specs-support/test.toml +++ b/acceptance/bundle/python/feature_specs-support/test.toml @@ -1,4 +1,4 @@ -Cloud = true +Cloud = false # tests don't interact with APIs # feature specs are only supported in the current version of the wheel EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/resources/feature_specs/basic/output.txt b/acceptance/bundle/resources/feature_specs/basic/output.txt index e07ac627f59..9a4a63d7e63 100644 --- a/acceptance/bundle/resources/feature_specs/basic/output.txt +++ b/acceptance/bundle/resources/feature_specs/basic/output.txt @@ -96,7 +96,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle summary -o json { "id": "[SPEC1_ID]", - "url": "[DATABRICKS_URL]/explore/data/functions/main/fspec_[UNIQUE_NAME]/myspec?w=[WORKSPACE_ID]" + "url": "[DATABRICKS_URL]/explore/data/functions/main/fspec_[UNIQUE_NAME]/myspec" } === A second plan on an unchanged bundle must report no changes @@ -116,8 +116,9 @@ Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-feature-specs-basic-[UNIQUE_NAME]/default/files... -This action will result in the deletion or recreation of the following feature specs. -Grants on them that are not declared in the bundle are lost: +This action will result in the deletion or recreation of the following FeatureSpecs. +Feature Serving endpoints and training code that use a deleted FeatureSpec can no longer +load it, and a recreated one loses the grants the bundle does not declare: recreate resources.feature_specs.spec1 Recreated feature_specs.spec1 Files: 1 uploaded, 0 deleted @@ -170,7 +171,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged The following resources will be deleted: delete resources.feature_specs.spec1 -This action will result in the deletion of the following feature specs. +This action will result in the deletion of the following FeatureSpecs. Feature Serving endpoints and training code that use them can no longer load them: delete resources.feature_specs.spec1 diff --git a/acceptance/bundle/resources/feature_specs/order/output.txt b/acceptance/bundle/resources/feature_specs/order/output.txt index 024250568df..9a077786021 100644 --- a/acceptance/bundle/resources/feature_specs/order/output.txt +++ b/acceptance/bundle/resources/feature_specs/order/output.txt @@ -28,7 +28,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged The following resources will be deleted: delete resources.feature_specs.spec1 -This action will result in the deletion of the following feature specs. +This action will result in the deletion of the following FeatureSpecs. Feature Serving endpoints and training code that use them can no longer load them: delete resources.feature_specs.spec1 diff --git a/acceptance/bundle/resources/feature_specs/remote-delete/output.txt b/acceptance/bundle/resources/feature_specs/remote-delete/output.txt index 4fdddf0b298..d08e36c2519 100644 --- a/acceptance/bundle/resources/feature_specs/remote-delete/output.txt +++ b/acceptance/bundle/resources/feature_specs/remote-delete/output.txt @@ -14,8 +14,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Delete the feature spec out of band ->>> [CLI] api delete /api/2.0/feature-engineering/feature-specs/main.fspec_[UNIQUE_NAME].myspec -{} +>>> [CLI] feature-engineering delete-feature-spec main.fspec_[UNIQUE_NAME].myspec === Plan should detect the feature spec is gone and re-create it diff --git a/acceptance/bundle/resources/feature_specs/remote-delete/script b/acceptance/bundle/resources/feature_specs/remote-delete/script index a0142c4c72d..4ab3f35ba8d 100644 --- a/acceptance/bundle/resources/feature_specs/remote-delete/script +++ b/acceptance/bundle/resources/feature_specs/remote-delete/script @@ -3,7 +3,7 @@ feature_spec_setup trace $CLI bundle deploy title "Delete the feature spec out of band" -MSYS_NO_PATHCONV=1 trace $CLI api delete /api/2.0/feature-engineering/feature-specs/$SCHEMA_FULL_NAME.myspec +trace $CLI feature-engineering delete-feature-spec $SCHEMA_FULL_NAME.myspec title "Plan should detect the feature spec is gone and re-create it" trace $CLI bundle plan diff --git a/acceptance/bundle/resources/feature_specs/test.toml b/acceptance/bundle/resources/feature_specs/test.toml index eebb76502b2..3fc5a9008c6 100644 --- a/acceptance/bundle/resources/feature_specs/test.toml +++ b/acceptance/bundle/resources/feature_specs/test.toml @@ -5,7 +5,8 @@ Cloud = true # mirrors Universe. EnvMatrixExclude.no_feature_spec_dms = ["DMS=true"] -# Fake SQL endpoint so create_feature_spec_fixtures.py works against the local test server too. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' +# A summary URL has ?w= only when the host does not already carry the workspace ID. +[[Repls]] +Old = '\?[ow]=\[WORKSPACE_ID\]' +New = '' +Order = 1000 diff --git a/acceptance/bundle/resources/feature_specs/uppercase-name/output.txt b/acceptance/bundle/resources/feature_specs/uppercase-name/output.txt index 635eb197427..0331176a238 100644 --- a/acceptance/bundle/resources/feature_specs/uppercase-name/output.txt +++ b/acceptance/bundle/resources/feature_specs/uppercase-name/output.txt @@ -69,8 +69,9 @@ Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-feature-specs-uppercase-name-[UNIQUE_NAME]/default/files... -This action will result in the deletion or recreation of the following feature specs. -Grants on them that are not declared in the bundle are lost: +This action will result in the deletion or recreation of the following FeatureSpecs. +Feature Serving endpoints and training code that use a deleted FeatureSpec can no longer +load it, and a recreated one loses the grants the bundle does not declare: recreate resources.feature_specs.spec1 Recreated feature_specs.spec1 Files: 1 uploaded, 0 deleted @@ -106,7 +107,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged The following resources will be deleted: delete resources.feature_specs.spec1 -This action will result in the deletion of the following feature specs. +This action will result in the deletion of the following FeatureSpecs. Feature Serving endpoints and training code that use them can no longer load them: delete resources.feature_specs.spec1 diff --git a/acceptance/bundle/resources/features/lifecycle/test.toml b/acceptance/bundle/resources/features/lifecycle/test.toml index 9c98d047fde..a26bade48d0 100644 --- a/acceptance/bundle/resources/features/lifecycle/test.toml +++ b/acceptance/bundle/resources/features/lifecycle/test.toml @@ -4,8 +4,3 @@ Ignore = [ ".databricks", "databricks.yml", ] - -# Fake SQL endpoint so create_table.py works against the local test server too. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' diff --git a/acceptance/bundle/resources/grants/feature_specs/output.txt b/acceptance/bundle/resources/grants/feature_specs/output.txt index a7b6b884b83..d4d1ce9e7bc 100644 --- a/acceptance/bundle/resources/grants/feature_specs/output.txt +++ b/acceptance/bundle/resources/grants/feature_specs/output.txt @@ -40,8 +40,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-feature-spec-grants-[UNIQUE_NAME]/default/files... -This action will result in the deletion or recreation of the following feature specs. -Grants on them that are not declared in the bundle are lost: +This action will result in the deletion or recreation of the following FeatureSpecs. +Feature Serving endpoints and training code that use a deleted FeatureSpec can no longer +load it, and a recreated one loses the grants the bundle does not declare: recreate resources.feature_specs.my_feature_spec Recreated feature_specs.my_feature_spec Updated feature_specs.my_feature_spec.grants @@ -66,7 +67,7 @@ Resources: 1 created, 1 changed, 1 deleted, 0 unchanged The following resources will be deleted: delete resources.feature_specs.my_feature_spec -This action will result in the deletion of the following feature specs. +This action will result in the deletion of the following FeatureSpecs. Feature Serving endpoints and training code that use them can no longer load them: delete resources.feature_specs.my_feature_spec diff --git a/acceptance/bundle/resources/grants/feature_specs/script b/acceptance/bundle/resources/grants/feature_specs/script index bc39ede6df8..8980c6d7b34 100644 --- a/acceptance/bundle/resources/grants/feature_specs/script +++ b/acceptance/bundle/resources/grants/feature_specs/script @@ -1,18 +1,6 @@ -# Creating a feature spec resolves its lookup tables in Unity Catalog, so the test creates them in -# a schema of its own. -export SCHEMA_FULL_NAME="main.fspec_$UNIQUE_NAME" -trace $CLI schemas create "fspec_$UNIQUE_NAME" main -o json | jq '{full_name}' -trace create_feature_spec_fixtures.py "$SCHEMA_FULL_NAME" +feature_spec_setup -cleanup() { - $CLI bundle destroy --auto-approve >> LOG.cleanup 2>&1 || true - $CLI schemas delete "$SCHEMA_FULL_NAME" --force >> LOG.cleanup 2>&1 || true -} -trap cleanup EXIT - -envsubst < databricks.yml.tmpl > databricks.yml - -trace $CLI bundle plan -o json | jq 'del(.features)' > out.plan1.json +trace $CLI bundle plan -o json | nostamp > out.plan1.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy1.requests.json @@ -20,7 +8,7 @@ trace $CLI grants get function $SCHEMA_FULL_NAME.myspec | jq --sort-keys title "Every change recreates the feature spec, which drops its grants; the bundle must re-apply them" trace update_file.py databricks.yml "[age, country]" "[age]" -trace $CLI bundle plan -o json | jq 'del(.features)' > out.plan2.json +trace $CLI bundle plan -o json | nostamp > out.plan2.json trace $CLI bundle deploy --auto-approve trace print_requests.py //feature-engineering //permissions > out.deploy2.requests.json trace $CLI grants get function $SCHEMA_FULL_NAME.myspec | jq --sort-keys diff --git a/acceptance/bundle/resources/grants/feature_specs/test.toml b/acceptance/bundle/resources/grants/feature_specs/test.toml index 08fbace3299..5aa0b291973 100644 --- a/acceptance/bundle/resources/grants/feature_specs/test.toml +++ b/acceptance/bundle/resources/grants/feature_specs/test.toml @@ -1,8 +1,3 @@ # Deployment history has no "feature_specs." resource-key prefix; see dmsResourceTypes in # libs/testserver/bundledeployments.go. EnvMatrixExclude.no_feature_spec_dms = ["DMS=true"] - -# Fake SQL endpoint so create_feature_spec_fixtures.py works against the local test server too. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' diff --git a/acceptance/bundle/resources/feature_specs/script.prepare b/acceptance/bundle/script.prepare similarity index 51% rename from acceptance/bundle/resources/feature_specs/script.prepare rename to acceptance/bundle/script.prepare index 0529e3ea8d2..26df36f925a 100644 --- a/acceptance/bundle/resources/feature_specs/script.prepare +++ b/acceptance/bundle/script.prepare @@ -1,5 +1,12 @@ -# Creating a feature spec resolves its lookup tables and UDFs in Unity Catalog, so each test creates -# them in a schema of its own. Dropping the schema also drops any feature spec a failed run leaves. +# Creates schema main.$1, exported as SCHEMA_FULL_NAME, holding the lookup tables and UDFs that +# creating a feature spec resolves in Unity Catalog. +feature_spec_fixtures() { + export SCHEMA_FULL_NAME="main.$1" + trace $CLI schemas create "$1" main -o json | jq '{full_name}' + trace create_feature_spec_fixtures.py "$SCHEMA_FULL_NAME" +} + +# Dropping the schema also drops any feature spec a failed run leaves. feature_spec_cleanup() { $CLI bundle destroy --auto-approve >> LOG.cleanup 2>&1 || true $CLI schemas delete "$SCHEMA_FULL_NAME" --force >> LOG.cleanup 2>&1 || true @@ -7,9 +14,7 @@ feature_spec_cleanup() { # Call from the test's script, so the trap belongs to the shell that runs it. feature_spec_setup() { - export SCHEMA_FULL_NAME="main.fspec_$UNIQUE_NAME" - trace $CLI schemas create "fspec_$UNIQUE_NAME" main -o json | jq '{full_name}' - trace create_feature_spec_fixtures.py "$SCHEMA_FULL_NAME" + feature_spec_fixtures "fspec_$UNIQUE_NAME" trap feature_spec_cleanup EXIT envsubst < databricks.yml.tmpl > databricks.yml } diff --git a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go index dd54bae891e..371c0323eb8 100644 --- a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go +++ b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go @@ -116,6 +116,22 @@ func resolveAiGatewayParent(b *bundle.Bundle, parent string) string { return "schemas/" + resolveCatalog(b, catalogName) + "." + resolveSchema(b, catalogName, schemaName) } +// resolveThreePartName applies resolveCatalog and resolveSchema to the catalog and +// schema of a "catalog.schema.name" UC identifier. A name that is not three parts, or +// that already contains a reference, is returned unchanged; a mix of references and +// literals is not supported. +func resolveThreePartName(b *bundle.Bundle, name string) string { + if dynvar.ContainsVariableReference(name) { + return name + } + parts, ok := splitUCName(name, 3) + if !ok { + return name + } + catalogName, schemaName := parts[0], parts[1] + return resolveCatalog(b, catalogName) + "." + resolveSchema(b, catalogName, schemaName) + "." + parts[2] +} + func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Resolve resources that depend on schemas before resolving schemas themselves. // The schema resolution below modifies schema.CatalogName, and findSchema @@ -175,32 +191,13 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia if idx == nil { continue } - // A name that already contains a reference is left as is; a mix of - // references and literals is not supported. - if dynvar.ContainsVariableReference(idx.Name) { - continue - } - // Name is a three-part "catalog.schema.index" UC identifier. - parts, ok := splitUCName(idx.Name, 3) - if !ok { - continue - } - catalogName, schemaName := parts[0], parts[1] - idx.Name = resolveCatalog(b, catalogName) + "." + resolveSchema(b, catalogName, schemaName) + "." + parts[2] + idx.Name = resolveThreePartName(b, idx.Name) } for _, f := range b.Config.Resources.Features { if f == nil { continue } - if dynvar.ContainsVariableReference(f.FullName) { - continue - } - parts, ok := splitUCName(f.FullName, 3) - if !ok { - continue - } - catalogName, schemaName := parts[0], parts[1] - f.FullName = resolveCatalog(b, catalogName) + "." + resolveSchema(b, catalogName, schemaName) + "." + parts[2] + f.FullName = resolveThreePartName(b, f.FullName) } for _, set := range b.Config.Resources.MaterializationSets { if set == nil { @@ -222,15 +219,7 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia if fs == nil { continue } - if dynvar.ContainsVariableReference(fs.Name) { - continue - } - parts, ok := splitUCName(fs.Name, 3) - if !ok { - continue - } - catalogName, schemaName := parts[0], parts[1] - fs.Name = resolveCatalog(b, catalogName) + "." + resolveSchema(b, catalogName, schemaName) + "." + parts[2] + fs.Name = resolveThreePartName(b, fs.Name) } for _, mse := range b.Config.Resources.ModelServingEndpoints { if mse == nil { diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 605f530a94c..7185ec0abf5 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" "testing" + "time" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" @@ -22,6 +23,7 @@ import ( "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/common/types/duration" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/dashboards" @@ -150,15 +152,28 @@ var testConfig map[string]any = map[string]any{ }, }, + // The fake accepts a declarative feature alongside lookups and functions, which the real API + // rejects. "feature_specs": &resources.FeatureSpec{ Name: "main.default.my_feature_spec", - Features: []ml.Features{{ - FeatureLookup: &ml.FeatureLookup{ - Table: "main.default.my_table", - LookupKeys: []string{"id"}, - FeatureNames: []string{"amount"}, - }, - }}, + Features: []ml.Features{ + {FeatureLookup: &ml.FeatureLookup{ + Table: "main.default.my_table", + LookupKeys: []string{"id"}, + TimestampLookupKeys: []string{"ts"}, + LookbackWindow: duration.New(time.Hour), + FeatureNames: []string{"amount", "country"}, + RenameOutputs: []ml.RenameOutput{{Feature: "amount", Output: "total_amount"}}, + DefaultValues: []ml.DefaultValue{{Feature: "country", DefaultValue: "unknown"}}, + }}, + {FeatureFunction: &ml.FeatureFunction{ + Udf: "main.default.my_udf", + Output: "score", + InputBindings: []ml.FeatureFunctionInputBinding{{Parameter: "x", BoundFeature: "amount"}}, + }}, + {DeclarativeFeature: &ml.DeclarativeFeature{Feature: "main.default.my_feature"}}, + }, + ExcludeColumns: []string{"country"}, ExtraPipRequirements: &ml.PipRequirements{Requirements: []string{"numpy==2.1.0"}}, }, diff --git a/bundle/direct/dresources/feature_spec.go b/bundle/direct/dresources/feature_spec.go index 078e26e3b4b..48d071d1127 100644 --- a/bundle/direct/dresources/feature_spec.go +++ b/bundle/direct/dresources/feature_spec.go @@ -20,8 +20,8 @@ func (*ResourceFeatureSpec) New(client *databricks.WorkspaceClient) *ResourceFea } // PrepareState sorts the fields in FeatureSpec that are order-free lists, so a reorder in config is -// not a recreate. The FeatureSpec with the order-free list fields sorted is what gets sent and -// stored in the backend. +// not a recreate. The sorted copy is what is sent in the create request and persisted in state; the +// config is not modified. func (*ResourceFeatureSpec) PrepareState(input *resources.FeatureSpec) *ml.FeatureSpec { state := input.FeatureSpec sortOrderFreeLists(&state) @@ -33,8 +33,8 @@ func (r *ResourceFeatureSpec) DoRead(ctx context.Context, id string) (*ml.Featur if err != nil { return nil, err } - // Returns the FeatureSpec with its order-free list fields sorted, so a feature spec created - // outside the bundle also has them sorted. + // Sorted the same way as in PrepareState, so the two compare equal whatever order the API + // returns the lists in. sortOrderFreeLists(featureSpec) return featureSpec, nil } @@ -75,8 +75,8 @@ func featureFunctionInputBindingKey(x ml.FeatureFunctionInputBinding) (string, s } // sortOrderFreeLists sorts a FeatureSpec's ExcludeColumns and ExtraPipRequirements. Both are -// treated as sets by the server but stored as string lists, which cannot be compared by a key -// defined in KeyedSlices. Each field is replaced with the sorted list rather than sorted in place, +// treated as sets by the server but stored as string lists, so they are sorted on both sides, as +// normalizeAssignments in grants.go does for privileges. Each field is replaced with a sorted copy, // so the bundle config is not mutated. func sortOrderFreeLists(featureSpec *ml.FeatureSpec) { featureSpec.ExcludeColumns = sortedClone(featureSpec.ExcludeColumns) @@ -87,7 +87,8 @@ func sortOrderFreeLists(featureSpec *ml.FeatureSpec) { } } -// sortedClone keeps an empty list empty rather than nil, so the request body is unchanged. +// sortedClone returns a sorted copy of values. An empty list stays empty rather than nil, so the +// request body is unchanged. func sortedClone(values []string) []string { sorted := slices.Clone(values) slices.Sort(sorted) diff --git a/bundle/phases/messages.go b/bundle/phases/messages.go index 0e7a29a9c82..535d8086b59 100644 --- a/bundle/phases/messages.go +++ b/bundle/phases/messages.go @@ -63,8 +63,9 @@ or recreate the Feature View. Deleting or recreating a Feature View that a Featu not blocked, and leaves the FeatureSpec stale:` deleteOrRecreateFeatureSpecMessage = ` -This action will result in the deletion or recreation of the following feature specs. -Grants on them that are not declared in the bundle are lost:` +This action will result in the deletion or recreation of the following FeatureSpecs. +Feature Serving endpoints and training code that use a deleted FeatureSpec can no longer +load it, and a recreated one loses the grants the bundle does not declare:` ) // DataLossWarning is the warning shown when a non-interactive command is about @@ -119,6 +120,6 @@ The conversation history attached to them is permanently lost:` deleteFeatureMessage = `This action will result in the deletion of the following Feature Views. Deleting a Feature View that a FeatureSpec serves is not blocked, and leaves the FeatureSpec stale:` - deleteFeatureSpecMessage = `This action will result in the deletion of the following feature specs. + deleteFeatureSpecMessage = `This action will result in the deletion of the following FeatureSpecs. Feature Serving endpoints and training code that use them can no longer load them:` ) diff --git a/libs/testserver/feature_spec.go b/libs/testserver/feature_spec.go index 61670dbd0d2..db8b92c0599 100644 --- a/libs/testserver/feature_spec.go +++ b/libs/testserver/feature_spec.go @@ -26,12 +26,6 @@ func (s *FakeWorkspace) FeatureSpecsCreate(req Request) Response { // UC normalizes feature spec names to lowercase. featureSpec.Name = strings.ToLower(featureSpec.Name) - if len(strings.Split(featureSpec.Name, ".")) != 3 { - return Response{ - StatusCode: http.StatusBadRequest, - Body: fmt.Sprintf("name must be a three-part name, got %q", featureSpec.Name), - } - } if _, exists := s.FeatureSpecs[featureSpec.Name]; exists { return Response{ StatusCode: http.StatusConflict, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 8d7708af5af..3ee64f8de25 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -11,10 +11,12 @@ import ( "mime/multipart" "net/http" "path" + "regexp" "strconv" "strings" "github.com/databricks/cli/libs/notebook" + "github.com/databricks/cli/libs/testserver/testsql" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -1168,10 +1170,19 @@ func AddDefaultHandlers(server *Server) { // error, custom status) can register its own handler for the same pattern // before calling AddDefaultHandlers, or stub it via test.toml in acceptance. server.Handle("POST", "/api/2.0/sql/statements", server.sqlExecuteStatement) + // The API also accepts a trailing slash, which acceptance scripts send. + server.Handle("POST", "/api/2.0/sql/statements/", server.sqlExecuteStatement) server.Handle("GET", "/api/2.0/sql/statements/{statement_id}", server.sqlGetStatement) server.Handle("GET", "/api/2.0/sql/statements/{statement_id}/result/chunks/{chunk_index}", server.sqlGetStatementResultChunk) server.Handle("POST", "/api/2.0/sql/statements/{statement_id}/cancel", server.sqlCancelStatement) + // The fake keeps no tables or functions, so the statements acceptance scripts + // run to set them up succeed without effect. + fixtureStatement := regexp.MustCompile(`(?i)^(CREATE\s+(TABLE|FUNCTION)|INSERT\s+INTO)\s`) + server.sqlHandler.HandleFallbackPattern(fixtureStatement, func(testsql.Request) testsql.Result { + return testsql.Result{} + }) + server.Handle("GET", "/api/2.0/preview/sql/data_sources", func(req Request) any { return req.Workspace.SqlDataSourcesList(req) }) diff --git a/libs/testserver/statements_test.go b/libs/testserver/statements_test.go index 5552f64f558..83059b3d5ef 100644 --- a/libs/testserver/statements_test.go +++ b/libs/testserver/statements_test.go @@ -1,7 +1,9 @@ package testserver import ( + "bytes" "encoding/json" + "net/http" "regexp" "testing" @@ -49,3 +51,56 @@ func TestHandleSQLPattern(t *testing.T) { require.NotNil(t, resp.Result) assert.Equal(t, [][]string{{"42"}}, resp.Result.DataArray) } + +// postSQL submits a statement over HTTP to path and returns the response. +func postSQL(t *testing.T, server *Server, path, statement string) *sql.StatementResponse { + t.Helper() + body, err := json.Marshal(sql.ExecuteStatementRequest{Statement: statement, WarehouseId: "wh-1", WaitTimeout: "10s"}) + require.NoError(t, err) + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, server.URL+path, bytes.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer token") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + var out sql.StatementResponse + require.NoError(t, json.NewDecoder(resp.Body).Decode(&out)) + return &out +} + +func TestDefaultHandlersAcceptFixtureStatements(t *testing.T) { + server := New(t) + AddDefaultHandlers(server) + + tests := []struct { + path string + statement string + want sql.StatementState + }{ + {"/api/2.0/sql/statements", "CREATE TABLE main.s.t (id INT)", sql.StatementStateSucceeded}, + {"/api/2.0/sql/statements/", "CREATE TABLE IF NOT EXISTS main.s.t AS SELECT 1", sql.StatementStateSucceeded}, + {"/api/2.0/sql/statements/", "create function main.s.f(x INT) RETURNS INT RETURN x", sql.StatementStateSucceeded}, + {"/api/2.0/sql/statements/", "INSERT INTO main.s.t VALUES (1)", sql.StatementStateSucceeded}, + {"/api/2.0/sql/statements/", "DROP TABLE main.s.t", sql.StatementStateFailed}, + } + for _, tc := range tests { + t.Run(tc.statement, func(t *testing.T) { + resp := postSQL(t, server, tc.path, tc.statement) + require.NotNil(t, resp.Status) + assert.Equal(t, tc.want, resp.Status.State) + }) + } +} + +func TestDefaultFixtureMatcherYieldsToTestMatcher(t *testing.T) { + server := New(t) + AddDefaultHandlers(server) + server.HandleSQL("CREATE TABLE main.s.t (id INT)", func(testsql.Request) testsql.Result { + return testsql.Result{Columns: []string{"a"}, Rows: [][]string{{"1"}}} + }) + + resp := postSQL(t, server, "/api/2.0/sql/statements", "CREATE TABLE main.s.t (id INT)") + require.NotNil(t, resp.Result) + assert.Equal(t, [][]string{{"1"}}, resp.Result.DataArray) +} diff --git a/libs/testserver/testsql/testsql.go b/libs/testserver/testsql/testsql.go index 0d28785f40f..2ecc0923d16 100644 --- a/libs/testserver/testsql/testsql.go +++ b/libs/testserver/testsql/testsql.go @@ -16,6 +16,7 @@ package testsql import ( "fmt" "regexp" + "slices" "strings" "sync" @@ -91,6 +92,7 @@ type statement struct { type Handler struct { mu sync.Mutex matchers []matcher + fallbacks []matcher statements map[string]*statement nextID int } @@ -123,12 +125,25 @@ func (h *Handler) Handle(statement string, fn func(Request) Result) { func (h *Handler) HandlePattern(re *regexp.Regexp, fn func(Request) Result) { h.mu.Lock() defer h.mu.Unlock() - h.matchers = append(h.matchers, matcher{ + h.matchers = append(h.matchers, patternMatcher(re, fn)) +} + +// HandleFallbackPattern registers a matcher like HandlePattern, but tried only +// when no Handle or HandlePattern matcher matches, whatever the registration +// order. +func (h *Handler) HandleFallbackPattern(re *regexp.Regexp, fn func(Request) Result) { + h.mu.Lock() + defer h.mu.Unlock() + h.fallbacks = append(h.fallbacks, patternMatcher(re, fn)) +} + +func patternMatcher(re *regexp.Regexp, fn func(Request) Result) matcher { + return matcher{ match: func(stmt string) []string { return re.FindStringSubmatch(stmt) }, fn: fn, - }) + } } // Submit matches the statement, runs the matcher once to capture its plan, @@ -215,7 +230,7 @@ func (h *Handler) plan(stmt string, params []sql.StatementParameterListItem) *st Code: sql.ServiceErrorCodeBadRequest, Message: "unsupported statement: " + stmt, }} - for _, m := range h.matchers { + for _, m := range slices.Concat(h.matchers, h.fallbacks) { groups := m.match(stmt) if groups == nil { continue diff --git a/libs/testserver/testsql/testsql_test.go b/libs/testserver/testsql/testsql_test.go index 7c70d038218..31228081396 100644 --- a/libs/testserver/testsql/testsql_test.go +++ b/libs/testserver/testsql/testsql_test.go @@ -55,6 +55,19 @@ func TestFirstMatchWins(t *testing.T) { assert.Equal(t, []string{"first"}, columnNames(resp)) } +func TestFallbackYieldsToMatchers(t *testing.T) { + e := testsql.New() + e.HandleFallbackPattern(regexp.MustCompile(`^CREATE`), func(testsql.Request) testsql.Result { + return testsql.Result{Columns: []string{"fallback"}} + }) + e.Handle("CREATE TABLE foo", func(testsql.Request) testsql.Result { + return testsql.Result{Columns: []string{"matcher"}} + }) + + assert.Equal(t, []string{"matcher"}, columnNames(e.Submit("CREATE TABLE foo", "10s", nil))) + assert.Equal(t, []string{"fallback"}, columnNames(e.Submit("CREATE TABLE bar", "10s", nil))) +} + func TestUnsupportedStatement(t *testing.T) { e := testsql.New() resp := e.Submit("SELECT 1", "10s", nil) From f09786230e141596651f1774b5a26bb9bb8ded51 Mon Sep 17 00:00:00 2001 From: Daniel Bai Date: Fri, 13 Nov 2026 18:27:27 +0000 Subject: [PATCH 2/2] Drop the SQL statement stubs the test server's fallback now answers The materialization set, quality monitor, Postgres synced table and synced database table tests only send CREATE TABLE and INSERT INTO statements, which the fallback answers. Co-authored-by: Isaac --- .../bundle/resources/materialization_sets/batch/test.toml | 5 ----- .../resources/materialization_sets/immutable/test.toml | 5 ----- .../resources/materialization_sets/membership/test.toml | 5 ----- .../resources/materialization_sets/mixed_triggers/test.toml | 5 ----- .../resources/materialization_sets/removal_scope/test.toml | 5 ----- acceptance/bundle/resources/postgres_synced_tables/test.toml | 5 ----- acceptance/bundle/resources/quality_monitors/test.toml | 5 ----- .../bundle/resources/synced_database_tables/basic/test.toml | 5 ----- 8 files changed, 40 deletions(-) diff --git a/acceptance/bundle/resources/materialization_sets/batch/test.toml b/acceptance/bundle/resources/materialization_sets/batch/test.toml index b7171278722..bc520ba96f9 100644 --- a/acceptance/bundle/resources/materialization_sets/batch/test.toml +++ b/acceptance/bundle/resources/materialization_sets/batch/test.toml @@ -13,11 +13,6 @@ Ignore = [ Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' New = '[DATABRICKS_URL]' -# Fake SQL endpoint so create_table.py works against the local test server too. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' - # The cleanup drops the source table; the fake has no delete handler for one. [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" diff --git a/acceptance/bundle/resources/materialization_sets/immutable/test.toml b/acceptance/bundle/resources/materialization_sets/immutable/test.toml index 9f9dbff911d..cb3b20fa154 100644 --- a/acceptance/bundle/resources/materialization_sets/immutable/test.toml +++ b/acceptance/bundle/resources/materialization_sets/immutable/test.toml @@ -21,11 +21,6 @@ New = '[UUID]' Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' New = '[DATABRICKS_URL]' -# Fake SQL endpoint so create_table.py works against the local test server too. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' - # The cleanup drops the source table; the fake has no delete handler for one. [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" diff --git a/acceptance/bundle/resources/materialization_sets/membership/test.toml b/acceptance/bundle/resources/materialization_sets/membership/test.toml index c6bdf296912..cecef384b26 100644 --- a/acceptance/bundle/resources/materialization_sets/membership/test.toml +++ b/acceptance/bundle/resources/materialization_sets/membership/test.toml @@ -23,11 +23,6 @@ New = '[UUID]' Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' New = '[DATABRICKS_URL]' -# Fake SQL endpoint so create_table.py works against the local test server too. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' - # The cleanup drops the source table; the fake has no delete handler for one. [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" diff --git a/acceptance/bundle/resources/materialization_sets/mixed_triggers/test.toml b/acceptance/bundle/resources/materialization_sets/mixed_triggers/test.toml index a2655f7d8db..c77379ee9ac 100644 --- a/acceptance/bundle/resources/materialization_sets/mixed_triggers/test.toml +++ b/acceptance/bundle/resources/materialization_sets/mixed_triggers/test.toml @@ -21,11 +21,6 @@ New = '[UUID]' Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' New = '[DATABRICKS_URL]' -# Fake SQL endpoint so create_table.py works against the local test server too. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' - # The cleanup drops the source table; the fake has no delete handler for one. [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/test.toml b/acceptance/bundle/resources/materialization_sets/removal_scope/test.toml index 266aab264f8..530da3b4e3e 100644 --- a/acceptance/bundle/resources/materialization_sets/removal_scope/test.toml +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/test.toml @@ -15,11 +15,6 @@ Ignore = [ Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' New = '[DATABRICKS_URL]' -# Fake SQL endpoint so create_table.py works against the local test server too. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' - # The cleanup drops the source table; the fake has no delete handler for one. [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" diff --git a/acceptance/bundle/resources/postgres_synced_tables/test.toml b/acceptance/bundle/resources/postgres_synced_tables/test.toml index 5fac75f29be..f139c44a610 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/test.toml @@ -11,11 +11,6 @@ Old = '\?[ow]=\[(WORKSPACE_ID|ALPHANUMID)\]' New = '' Order = 1000 -# Fake SQL endpoint for the per-test source table create in script. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' - [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" Response.Body = '{"status": "OK"}' diff --git a/acceptance/bundle/resources/quality_monitors/test.toml b/acceptance/bundle/resources/quality_monitors/test.toml index 4e383d30499..a4274f3bbaf 100644 --- a/acceptance/bundle/resources/quality_monitors/test.toml +++ b/acceptance/bundle/resources/quality_monitors/test.toml @@ -2,11 +2,6 @@ Cloud = true RecordRequests = true -# Fake SQL endpoint for local tests -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' - [[Repls]] Old = '"dashboard_id":\s*"[0-9a-f]+",' New = '"dashboard_id": "(redacted)",' diff --git a/acceptance/bundle/resources/synced_database_tables/basic/test.toml b/acceptance/bundle/resources/synced_database_tables/basic/test.toml index 41b51ae04a0..6715198c9d9 100644 --- a/acceptance/bundle/resources/synced_database_tables/basic/test.toml +++ b/acceptance/bundle/resources/synced_database_tables/basic/test.toml @@ -13,11 +13,6 @@ Old = '\?[ow]=\[(WORKSPACE_ID|ALPHANUMID)\]' New = '' Order = 1000 -# Fake SQL endpoint for local tests -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' - [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" Response.Body = '{"status": "OK"}'