diff --git a/acceptance/bin/create_feature_spec_fixtures.py b/acceptance/bin/create_feature_spec_fixtures.py index 78fe9ed9b05..4ca06250c45 100755 --- a/acceptance/bin/create_feature_spec_fixtures.py +++ b/acceptance/bin/create_feature_spec_fixtures.py @@ -6,44 +6,21 @@ Usage: create_feature_spec_fixtures.py CATALOG.SCHEMA """ -import json -import os -import subprocess import sys -import time - -schema_full_name = sys.argv[1] -cli = os.environ.get("CLI", "databricks") -warehouse_id = os.environ["TEST_DEFAULT_WAREHOUSE_ID"] - -statements = [ - f"CREATE TABLE {schema_full_name}.user_features " - "(user_id INT NOT NULL, age INT, country STRING, CONSTRAINT user_features_pk PRIMARY KEY (user_id))", - f"CREATE TABLE {schema_full_name}.tenant_user_features " - "(tenant_id INT NOT NULL, user_id INT NOT NULL, age INT, country STRING, " - "CONSTRAINT tenant_user_features_pk PRIMARY KEY (tenant_id, user_id))", - f"CREATE FUNCTION {schema_full_name}.score(x INT) RETURNS DOUBLE LANGUAGE PYTHON AS $$ return float(x) $$", - f"CREATE FUNCTION {schema_full_name}.country_score(x INT, y STRING) RETURNS DOUBLE LANGUAGE PYTHON " - "AS $$ return float(x) + len(y) $$", -] +from util import run_statements -def call_api(*args): - result = subprocess.run([cli, "api", *args], capture_output=True, text=True) - if result.returncode != 0: - print(f"databricks api {' '.join(args)} failed: {result.stderr}", file=sys.stderr) - sys.exit(1) - return json.loads(result.stdout) - +schema_full_name = sys.argv[1] -for statement in statements: - payload = json.dumps({"warehouse_id": warehouse_id, "statement": statement, "wait_timeout": "50s"}) - response = call_api("post", "/api/2.0/sql/statements/", "--json", payload) - # A warehouse that is still starting outlasts wait_timeout. - deadline = time.monotonic() + 600 - while response["status"]["state"] in ("PENDING", "RUNNING") and time.monotonic() < deadline: - time.sleep(5) - response = call_api("get", f"/api/2.0/sql/statements/{response['statement_id']}") - if response["status"]["state"] != "SUCCEEDED": - print(f"Statement {statement!r} ended in {response['status']}", file=sys.stderr) - sys.exit(1) +run_statements( + [ + f"CREATE TABLE {schema_full_name}.user_features " + "(user_id INT NOT NULL, age INT, country STRING, CONSTRAINT user_features_pk PRIMARY KEY (user_id))", + f"CREATE TABLE {schema_full_name}.tenant_user_features " + "(tenant_id INT NOT NULL, user_id INT NOT NULL, age INT, country STRING, " + "CONSTRAINT tenant_user_features_pk PRIMARY KEY (tenant_id, user_id))", + f"CREATE FUNCTION {schema_full_name}.score(x INT) RETURNS DOUBLE LANGUAGE PYTHON AS $$ return float(x) $$", + f"CREATE FUNCTION {schema_full_name}.country_score(x INT, y STRING) RETURNS DOUBLE LANGUAGE PYTHON " + "AS $$ return float(x) + len(y) $$", + ] +) diff --git a/acceptance/bin/create_table.py b/acceptance/bin/create_table.py index 1fb028dda18..fb04c4cfd8b 100755 --- a/acceptance/bin/create_table.py +++ b/acceptance/bin/create_table.py @@ -5,6 +5,8 @@ import sys import time +from util import run_statements + table_name = sys.argv[1] # Extract catalog.schema from table_name parts = table_name.split(".") @@ -24,36 +26,19 @@ def run_cli(*args): return result -def execute_sql(warehouse_id, sql): - """Execute SQL using the API endpoint.""" - payload = json.dumps({"warehouse_id": warehouse_id, "statement": sql, "wait_timeout": "30s"}) - return run_cli("api", "post", "/api/2.0/sql/statements/", "--json", payload) - - -# Get warehouse ID from environment variable -warehouse_id = os.environ["TEST_DEFAULT_WAREHOUSE_ID"] - # Create a simple table -sql = f"CREATE TABLE IF NOT EXISTS {table_name} (id INT, value STRING, timestamp TIMESTAMP) USING DELTA" - -result = execute_sql(warehouse_id, sql) -if result.returncode != 0: - print(f"Failed to create table: {result.stderr}", file=sys.stderr) - sys.exit(1) - +run_statements([f"CREATE TABLE IF NOT EXISTS {table_name} (id INT, value STRING, timestamp TIMESTAMP) USING DELTA"]) print(f"Created table {table_name}") # Insert some sample data so the monitor has something to analyze -insert_sql = f"""INSERT INTO {table_name} VALUES +run_statements( + [ + f"""INSERT INTO {table_name} VALUES (1, 'test1', current_timestamp()), (2, 'test2', current_timestamp()), (3, 'test3', current_timestamp())""" - -result = execute_sql(warehouse_id, insert_sql) -if result.returncode != 0: - print(f"Failed to insert data: {result.stderr}", file=sys.stderr) - sys.exit(1) - + ] +) print(f"Inserted sample data into {table_name}") # Wait for table to be visible in Unity Catalog diff --git a/acceptance/bin/util.py b/acceptance/bin/util.py index 3ee8d65bc90..d7045670433 100644 --- a/acceptance/bin/util.py +++ b/acceptance/bin/util.py @@ -1,7 +1,9 @@ import json +import os import shlex import subprocess import sys +import time VERBOSE = False @@ -31,3 +33,19 @@ def run(cmd): if result.returncode != 0: raise RunError(f"{cmd} failed with code {result.returncode}") return result + + +def run_statements(statements): + """Runs SQL statements on the test warehouse one after another, waiting for each to succeed.""" + cli = os.environ.get("CLI", "databricks") + warehouse_id = os.environ["TEST_DEFAULT_WAREHOUSE_ID"] + for statement in statements: + payload = json.dumps({"warehouse_id": warehouse_id, "statement": statement, "wait_timeout": "50s"}) + response = run_json([cli, "api", "post", "/api/2.0/sql/statements/", "--json", payload]) + # A warehouse that is still starting outlasts wait_timeout. + deadline = time.monotonic() + 600 + while response["status"]["state"] in ("PENDING", "RUNNING") and time.monotonic() < deadline: + time.sleep(5) + response = run_json([cli, "api", "get", f"/api/2.0/sql/statements/{response['statement_id']}"]) + if response["status"]["state"] != "SUCCEEDED": + raise RunError(f"Statement {statement!r} ended in {response['status']}") diff --git a/acceptance/bundle/deployment/bind/feature_spec/output.txt b/acceptance/bundle/deployment/bind/feature_spec/output.txt index a983fb36577..a859db162c0 100644 --- a/acceptance/bundle/deployment/bind/feature_spec/output.txt +++ b/acceptance/bundle/deployment/bind/feature_spec/output.txt @@ -8,7 +8,7 @@ === Create the feature spec outside the bundle ->>> [CLI] api post /api/2.0/feature-engineering/feature-specs --json { +>>> [CLI] feature-engineering create-feature-spec --json { "name": "main.fspec_[UNIQUE_NAME].boundspec", "features": [{"feature_lookup": {"table": "main.fspec_[UNIQUE_NAME].user_features", "lookup_keys": ["user_id"], "feature_names": ["age"]}}] } @@ -51,5 +51,4 @@ Resources: Name: main.fspec_[UNIQUE_NAME].boundspec URL: (not deployed) ->>> [CLI] api delete /api/2.0/feature-engineering/feature-specs/main.fspec_[UNIQUE_NAME].boundspec -{} +>>> [CLI] feature-engineering delete-feature-spec main.fspec_[UNIQUE_NAME].boundspec diff --git a/acceptance/bundle/deployment/bind/feature_spec/script b/acceptance/bundle/deployment/bind/feature_spec/script index cbe4e43b07e..608c7fc8edc 100644 --- a/acceptance/bundle/deployment/bind/feature_spec/script +++ b/acceptance/bundle/deployment/bind/feature_spec/script @@ -1,18 +1,7 @@ -# Creating a feature spec resolves its lookup tables in Unity Catalog, so the test creates them in -# a schema of its own. Dropping the schema also drops the feature spec. -export SCHEMA_FULL_NAME="main.fspec_$UNIQUE_NAME" -trace $CLI schemas create "fspec_$UNIQUE_NAME" main -o json | jq '{full_name}' -trace create_feature_spec_fixtures.py "$SCHEMA_FULL_NAME" - -cleanup() { - $CLI schemas delete "$SCHEMA_FULL_NAME" --force >> LOG.cleanup 2>&1 || true -} -trap cleanup EXIT - -envsubst < databricks.yml.tmpl > databricks.yml +feature_spec_setup title "Create the feature spec outside the bundle" -MSYS_NO_PATHCONV=1 trace $CLI api post /api/2.0/feature-engineering/feature-specs --json "{ +trace $CLI feature-engineering create-feature-spec --json "{ \"name\": \"$SCHEMA_FULL_NAME.boundspec\", \"features\": [{\"feature_lookup\": {\"table\": \"$SCHEMA_FULL_NAME.user_features\", \"lookup_keys\": [\"user_id\"], \"feature_names\": [\"age\"]}}] }" | jq '{name}' @@ -26,4 +15,4 @@ trace $CLI bundle plan trace $CLI bundle deployment unbind spec1 trace $CLI bundle summary -MSYS_NO_PATHCONV=1 trace $CLI api delete /api/2.0/feature-engineering/feature-specs/$SCHEMA_FULL_NAME.boundspec +trace $CLI feature-engineering delete-feature-spec $SCHEMA_FULL_NAME.boundspec diff --git a/acceptance/bundle/deployment/bind/feature_spec/test.toml b/acceptance/bundle/deployment/bind/feature_spec/test.toml index e41de801cf3..66cb91f3551 100644 --- a/acceptance/bundle/deployment/bind/feature_spec/test.toml +++ b/acceptance/bundle/deployment/bind/feature_spec/test.toml @@ -2,11 +2,6 @@ Ignore = [ ".databricks", ] -# Fake SQL endpoint so create_feature_spec_fixtures.py works against the local test server too. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' - # Azure and GCP hosts carry the workspace ID, so their summary URLs have no ?w=. [[Repls]] Old = '\?[ow]=\[WORKSPACE_ID\]' diff --git a/acceptance/bundle/invariant/configs/feature_spec.yml.tmpl-init.sh b/acceptance/bundle/invariant/configs/feature_spec.yml.tmpl-init.sh index 91158f1d7ce..986943a23b5 100755 --- a/acceptance/bundle/invariant/configs/feature_spec.yml.tmpl-init.sh +++ b/acceptance/bundle/invariant/configs/feature_spec.yml.tmpl-init.sh @@ -2,7 +2,4 @@ # Creating a feature spec resolves its lookup table in Unity Catalog, and the invariant harness # provisions no tables of its own. -echo "Creating temporary schema and feature table: main.fspec_inv_$UNIQUE_NAME" - -$CLI schemas create fspec_inv_$UNIQUE_NAME main -o json | jq '{full_name}' -create_feature_spec_fixtures.py main.fspec_inv_$UNIQUE_NAME +feature_spec_fixtures "fspec_inv_$UNIQUE_NAME" diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index dccac6ed8bc..1c1abd19348 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -128,11 +128,6 @@ EnvMatrixExclude.no_data_quality_monitor_on_cloud = ["CONFIG_Cloud=true", "INPUT EnvMatrixExclude.no_feature_spec_dms = ["DMS=true", "INPUT_CONFIG=feature_spec.yml.tmpl"] # Same for the "postgres_data_apis." prefix. EnvMatrixExclude.no_postgres_data_api_dms = ["DMS=true", "INPUT_CONFIG=postgres_data_api.yml.tmpl"] -# Fake SQL endpoint for local tests - -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" diff --git a/acceptance/bundle/python/feature_specs-support/databricks.yml b/acceptance/bundle/python/feature_specs-support/databricks.yml index c4ded89a6eb..201ed896e2a 100644 --- a/acceptance/bundle/python/feature_specs-support/databricks.yml +++ b/acceptance/bundle/python/feature_specs-support/databricks.yml @@ -18,6 +18,3 @@ resources: table: main.default.user_features lookup_keys: [user_id] feature_names: [age] - grants: - - principal: account users - privileges: [EXECUTE] diff --git a/acceptance/bundle/python/feature_specs-support/out.test.toml b/acceptance/bundle/python/feature_specs-support/out.test.toml index ea77b758035..aeddf3e619e 100644 --- a/acceptance/bundle/python/feature_specs-support/out.test.toml +++ b/acceptance/bundle/python/feature_specs-support/out.test.toml @@ -1,3 +1,3 @@ -Cloud = true +Cloud = false EnvMatrix.DMS = ["", "true"] EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/python/feature_specs-support/output.txt b/acceptance/bundle/python/feature_specs-support/output.txt index db824f305b3..8982e0071c1 100644 --- a/acceptance/bundle/python/feature_specs-support/output.txt +++ b/acceptance/bundle/python/feature_specs-support/output.txt @@ -27,14 +27,6 @@ } } ], - "grants": [ - { - "principal": "account users", - "privileges": [ - "EXECUTE" - ] - } - ], "name": "main.default.my_feature_spec_1_updated" }, "my_feature_spec_2": { diff --git a/acceptance/bundle/python/feature_specs-support/test.toml b/acceptance/bundle/python/feature_specs-support/test.toml index 6aee90e30c0..d0aada9733d 100644 --- a/acceptance/bundle/python/feature_specs-support/test.toml +++ b/acceptance/bundle/python/feature_specs-support/test.toml @@ -1,4 +1,4 @@ -Cloud = true +Cloud = false # tests don't interact with APIs # feature specs are only supported in the current version of the wheel EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/resources/feature_specs/basic/output.txt b/acceptance/bundle/resources/feature_specs/basic/output.txt index e07ac627f59..9a4a63d7e63 100644 --- a/acceptance/bundle/resources/feature_specs/basic/output.txt +++ b/acceptance/bundle/resources/feature_specs/basic/output.txt @@ -96,7 +96,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle summary -o json { "id": "[SPEC1_ID]", - "url": "[DATABRICKS_URL]/explore/data/functions/main/fspec_[UNIQUE_NAME]/myspec?w=[WORKSPACE_ID]" + "url": "[DATABRICKS_URL]/explore/data/functions/main/fspec_[UNIQUE_NAME]/myspec" } === A second plan on an unchanged bundle must report no changes @@ -116,8 +116,9 @@ Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-feature-specs-basic-[UNIQUE_NAME]/default/files... -This action will result in the deletion or recreation of the following feature specs. -Grants on them that are not declared in the bundle are lost: +This action will result in the deletion or recreation of the following FeatureSpecs. +Feature Serving endpoints and training code that use a deleted FeatureSpec can no longer +load it, and a recreated one loses the grants the bundle does not declare: recreate resources.feature_specs.spec1 Recreated feature_specs.spec1 Files: 1 uploaded, 0 deleted @@ -170,7 +171,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged The following resources will be deleted: delete resources.feature_specs.spec1 -This action will result in the deletion of the following feature specs. +This action will result in the deletion of the following FeatureSpecs. Feature Serving endpoints and training code that use them can no longer load them: delete resources.feature_specs.spec1 diff --git a/acceptance/bundle/resources/feature_specs/order/output.txt b/acceptance/bundle/resources/feature_specs/order/output.txt index 024250568df..9a077786021 100644 --- a/acceptance/bundle/resources/feature_specs/order/output.txt +++ b/acceptance/bundle/resources/feature_specs/order/output.txt @@ -28,7 +28,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged The following resources will be deleted: delete resources.feature_specs.spec1 -This action will result in the deletion of the following feature specs. +This action will result in the deletion of the following FeatureSpecs. Feature Serving endpoints and training code that use them can no longer load them: delete resources.feature_specs.spec1 diff --git a/acceptance/bundle/resources/feature_specs/remote-delete/output.txt b/acceptance/bundle/resources/feature_specs/remote-delete/output.txt index 4fdddf0b298..d08e36c2519 100644 --- a/acceptance/bundle/resources/feature_specs/remote-delete/output.txt +++ b/acceptance/bundle/resources/feature_specs/remote-delete/output.txt @@ -14,8 +14,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Delete the feature spec out of band ->>> [CLI] api delete /api/2.0/feature-engineering/feature-specs/main.fspec_[UNIQUE_NAME].myspec -{} +>>> [CLI] feature-engineering delete-feature-spec main.fspec_[UNIQUE_NAME].myspec === Plan should detect the feature spec is gone and re-create it diff --git a/acceptance/bundle/resources/feature_specs/remote-delete/script b/acceptance/bundle/resources/feature_specs/remote-delete/script index a0142c4c72d..4ab3f35ba8d 100644 --- a/acceptance/bundle/resources/feature_specs/remote-delete/script +++ b/acceptance/bundle/resources/feature_specs/remote-delete/script @@ -3,7 +3,7 @@ feature_spec_setup trace $CLI bundle deploy title "Delete the feature spec out of band" -MSYS_NO_PATHCONV=1 trace $CLI api delete /api/2.0/feature-engineering/feature-specs/$SCHEMA_FULL_NAME.myspec +trace $CLI feature-engineering delete-feature-spec $SCHEMA_FULL_NAME.myspec title "Plan should detect the feature spec is gone and re-create it" trace $CLI bundle plan diff --git a/acceptance/bundle/resources/feature_specs/test.toml b/acceptance/bundle/resources/feature_specs/test.toml index eebb76502b2..3fc5a9008c6 100644 --- a/acceptance/bundle/resources/feature_specs/test.toml +++ b/acceptance/bundle/resources/feature_specs/test.toml @@ -5,7 +5,8 @@ Cloud = true # mirrors Universe. EnvMatrixExclude.no_feature_spec_dms = ["DMS=true"] -# Fake SQL endpoint so create_feature_spec_fixtures.py works against the local test server too. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' +# A summary URL has ?w= only when the host does not already carry the workspace ID. +[[Repls]] +Old = '\?[ow]=\[WORKSPACE_ID\]' +New = '' +Order = 1000 diff --git a/acceptance/bundle/resources/feature_specs/uppercase-name/output.txt b/acceptance/bundle/resources/feature_specs/uppercase-name/output.txt index 635eb197427..0331176a238 100644 --- a/acceptance/bundle/resources/feature_specs/uppercase-name/output.txt +++ b/acceptance/bundle/resources/feature_specs/uppercase-name/output.txt @@ -69,8 +69,9 @@ Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-feature-specs-uppercase-name-[UNIQUE_NAME]/default/files... -This action will result in the deletion or recreation of the following feature specs. -Grants on them that are not declared in the bundle are lost: +This action will result in the deletion or recreation of the following FeatureSpecs. +Feature Serving endpoints and training code that use a deleted FeatureSpec can no longer +load it, and a recreated one loses the grants the bundle does not declare: recreate resources.feature_specs.spec1 Recreated feature_specs.spec1 Files: 1 uploaded, 0 deleted @@ -106,7 +107,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged The following resources will be deleted: delete resources.feature_specs.spec1 -This action will result in the deletion of the following feature specs. +This action will result in the deletion of the following FeatureSpecs. Feature Serving endpoints and training code that use them can no longer load them: delete resources.feature_specs.spec1 diff --git a/acceptance/bundle/resources/features/lifecycle/test.toml b/acceptance/bundle/resources/features/lifecycle/test.toml index 9c98d047fde..a26bade48d0 100644 --- a/acceptance/bundle/resources/features/lifecycle/test.toml +++ b/acceptance/bundle/resources/features/lifecycle/test.toml @@ -4,8 +4,3 @@ Ignore = [ ".databricks", "databricks.yml", ] - -# Fake SQL endpoint so create_table.py works against the local test server too. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' diff --git a/acceptance/bundle/resources/grants/feature_specs/output.txt b/acceptance/bundle/resources/grants/feature_specs/output.txt index a7b6b884b83..d4d1ce9e7bc 100644 --- a/acceptance/bundle/resources/grants/feature_specs/output.txt +++ b/acceptance/bundle/resources/grants/feature_specs/output.txt @@ -40,8 +40,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-feature-spec-grants-[UNIQUE_NAME]/default/files... -This action will result in the deletion or recreation of the following feature specs. -Grants on them that are not declared in the bundle are lost: +This action will result in the deletion or recreation of the following FeatureSpecs. +Feature Serving endpoints and training code that use a deleted FeatureSpec can no longer +load it, and a recreated one loses the grants the bundle does not declare: recreate resources.feature_specs.my_feature_spec Recreated feature_specs.my_feature_spec Updated feature_specs.my_feature_spec.grants @@ -66,7 +67,7 @@ Resources: 1 created, 1 changed, 1 deleted, 0 unchanged The following resources will be deleted: delete resources.feature_specs.my_feature_spec -This action will result in the deletion of the following feature specs. +This action will result in the deletion of the following FeatureSpecs. Feature Serving endpoints and training code that use them can no longer load them: delete resources.feature_specs.my_feature_spec diff --git a/acceptance/bundle/resources/grants/feature_specs/script b/acceptance/bundle/resources/grants/feature_specs/script index bc39ede6df8..8980c6d7b34 100644 --- a/acceptance/bundle/resources/grants/feature_specs/script +++ b/acceptance/bundle/resources/grants/feature_specs/script @@ -1,18 +1,6 @@ -# Creating a feature spec resolves its lookup tables in Unity Catalog, so the test creates them in -# a schema of its own. -export SCHEMA_FULL_NAME="main.fspec_$UNIQUE_NAME" -trace $CLI schemas create "fspec_$UNIQUE_NAME" main -o json | jq '{full_name}' -trace create_feature_spec_fixtures.py "$SCHEMA_FULL_NAME" +feature_spec_setup -cleanup() { - $CLI bundle destroy --auto-approve >> LOG.cleanup 2>&1 || true - $CLI schemas delete "$SCHEMA_FULL_NAME" --force >> LOG.cleanup 2>&1 || true -} -trap cleanup EXIT - -envsubst < databricks.yml.tmpl > databricks.yml - -trace $CLI bundle plan -o json | jq 'del(.features)' > out.plan1.json +trace $CLI bundle plan -o json | nostamp > out.plan1.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy1.requests.json @@ -20,7 +8,7 @@ trace $CLI grants get function $SCHEMA_FULL_NAME.myspec | jq --sort-keys title "Every change recreates the feature spec, which drops its grants; the bundle must re-apply them" trace update_file.py databricks.yml "[age, country]" "[age]" -trace $CLI bundle plan -o json | jq 'del(.features)' > out.plan2.json +trace $CLI bundle plan -o json | nostamp > out.plan2.json trace $CLI bundle deploy --auto-approve trace print_requests.py //feature-engineering //permissions > out.deploy2.requests.json trace $CLI grants get function $SCHEMA_FULL_NAME.myspec | jq --sort-keys diff --git a/acceptance/bundle/resources/grants/feature_specs/test.toml b/acceptance/bundle/resources/grants/feature_specs/test.toml index 08fbace3299..5aa0b291973 100644 --- a/acceptance/bundle/resources/grants/feature_specs/test.toml +++ b/acceptance/bundle/resources/grants/feature_specs/test.toml @@ -1,8 +1,3 @@ # Deployment history has no "feature_specs." resource-key prefix; see dmsResourceTypes in # libs/testserver/bundledeployments.go. EnvMatrixExclude.no_feature_spec_dms = ["DMS=true"] - -# Fake SQL endpoint so create_feature_spec_fixtures.py works against the local test server too. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' diff --git a/acceptance/bundle/resources/materialization_sets/batch/test.toml b/acceptance/bundle/resources/materialization_sets/batch/test.toml index b7171278722..bc520ba96f9 100644 --- a/acceptance/bundle/resources/materialization_sets/batch/test.toml +++ b/acceptance/bundle/resources/materialization_sets/batch/test.toml @@ -13,11 +13,6 @@ Ignore = [ Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' New = '[DATABRICKS_URL]' -# Fake SQL endpoint so create_table.py works against the local test server too. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' - # The cleanup drops the source table; the fake has no delete handler for one. [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" diff --git a/acceptance/bundle/resources/materialization_sets/immutable/test.toml b/acceptance/bundle/resources/materialization_sets/immutable/test.toml index 3ca9d9f5485..594c4138d75 100644 --- a/acceptance/bundle/resources/materialization_sets/immutable/test.toml +++ b/acceptance/bundle/resources/materialization_sets/immutable/test.toml @@ -17,11 +17,6 @@ New = '[UUID]' Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' New = '[DATABRICKS_URL]' -# Fake SQL endpoint so create_table.py works against the local test server too. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' - # The cleanup drops the source table; the fake has no delete handler for one. [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" diff --git a/acceptance/bundle/resources/materialization_sets/membership/test.toml b/acceptance/bundle/resources/materialization_sets/membership/test.toml index ce3d6d1a504..ade42d6a1a8 100644 --- a/acceptance/bundle/resources/materialization_sets/membership/test.toml +++ b/acceptance/bundle/resources/materialization_sets/membership/test.toml @@ -19,11 +19,6 @@ New = '[UUID]' Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' New = '[DATABRICKS_URL]' -# Fake SQL endpoint so create_table.py works against the local test server too. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' - # The cleanup drops the source table; the fake has no delete handler for one. [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" diff --git a/acceptance/bundle/resources/materialization_sets/mixed_triggers/test.toml b/acceptance/bundle/resources/materialization_sets/mixed_triggers/test.toml index a2655f7d8db..c77379ee9ac 100644 --- a/acceptance/bundle/resources/materialization_sets/mixed_triggers/test.toml +++ b/acceptance/bundle/resources/materialization_sets/mixed_triggers/test.toml @@ -21,11 +21,6 @@ New = '[UUID]' Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' New = '[DATABRICKS_URL]' -# Fake SQL endpoint so create_table.py works against the local test server too. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' - # The cleanup drops the source table; the fake has no delete handler for one. [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/test.toml b/acceptance/bundle/resources/materialization_sets/removal_scope/test.toml index 3d6d20136c0..64a2c6fe929 100644 --- a/acceptance/bundle/resources/materialization_sets/removal_scope/test.toml +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/test.toml @@ -11,11 +11,6 @@ Ignore = [ Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' New = '[DATABRICKS_URL]' -# Fake SQL endpoint so create_table.py works against the local test server too. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' - # The cleanup drops the source table; the fake has no delete handler for one. [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" diff --git a/acceptance/bundle/resources/postgres_synced_tables/test.toml b/acceptance/bundle/resources/postgres_synced_tables/test.toml index 5fac75f29be..f139c44a610 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/test.toml @@ -11,11 +11,6 @@ Old = '\?[ow]=\[(WORKSPACE_ID|ALPHANUMID)\]' New = '' Order = 1000 -# Fake SQL endpoint for the per-test source table create in script. -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' - [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" Response.Body = '{"status": "OK"}' diff --git a/acceptance/bundle/resources/quality_monitors/test.toml b/acceptance/bundle/resources/quality_monitors/test.toml index 9a99197e216..c13813aa16a 100644 --- a/acceptance/bundle/resources/quality_monitors/test.toml +++ b/acceptance/bundle/resources/quality_monitors/test.toml @@ -1,8 +1,3 @@ # Test quality monitor output_schema_name updates Cloud = true RecordRequests = true - -# Fake SQL endpoint for local tests -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' diff --git a/acceptance/bundle/resources/synced_database_tables/basic/test.toml b/acceptance/bundle/resources/synced_database_tables/basic/test.toml index 41b51ae04a0..6715198c9d9 100644 --- a/acceptance/bundle/resources/synced_database_tables/basic/test.toml +++ b/acceptance/bundle/resources/synced_database_tables/basic/test.toml @@ -13,11 +13,6 @@ Old = '\?[ow]=\[(WORKSPACE_ID|ALPHANUMID)\]' New = '' Order = 1000 -# Fake SQL endpoint for local tests -[[Server]] -Pattern = "POST /api/2.0/sql/statements/" -Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' - [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" Response.Body = '{"status": "OK"}' diff --git a/acceptance/bundle/resources/feature_specs/script.prepare b/acceptance/bundle/script.prepare similarity index 51% rename from acceptance/bundle/resources/feature_specs/script.prepare rename to acceptance/bundle/script.prepare index 0529e3ea8d2..26df36f925a 100644 --- a/acceptance/bundle/resources/feature_specs/script.prepare +++ b/acceptance/bundle/script.prepare @@ -1,5 +1,12 @@ -# Creating a feature spec resolves its lookup tables and UDFs in Unity Catalog, so each test creates -# them in a schema of its own. Dropping the schema also drops any feature spec a failed run leaves. +# Creates schema main.$1, exported as SCHEMA_FULL_NAME, holding the lookup tables and UDFs that +# creating a feature spec resolves in Unity Catalog. +feature_spec_fixtures() { + export SCHEMA_FULL_NAME="main.$1" + trace $CLI schemas create "$1" main -o json | jq '{full_name}' + trace create_feature_spec_fixtures.py "$SCHEMA_FULL_NAME" +} + +# Dropping the schema also drops any feature spec a failed run leaves. feature_spec_cleanup() { $CLI bundle destroy --auto-approve >> LOG.cleanup 2>&1 || true $CLI schemas delete "$SCHEMA_FULL_NAME" --force >> LOG.cleanup 2>&1 || true @@ -7,9 +14,7 @@ feature_spec_cleanup() { # Call from the test's script, so the trap belongs to the shell that runs it. feature_spec_setup() { - export SCHEMA_FULL_NAME="main.fspec_$UNIQUE_NAME" - trace $CLI schemas create "fspec_$UNIQUE_NAME" main -o json | jq '{full_name}' - trace create_feature_spec_fixtures.py "$SCHEMA_FULL_NAME" + feature_spec_fixtures "fspec_$UNIQUE_NAME" trap feature_spec_cleanup EXIT envsubst < databricks.yml.tmpl > databricks.yml } diff --git a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go index dd54bae891e..371c0323eb8 100644 --- a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go +++ b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go @@ -116,6 +116,22 @@ func resolveAiGatewayParent(b *bundle.Bundle, parent string) string { return "schemas/" + resolveCatalog(b, catalogName) + "." + resolveSchema(b, catalogName, schemaName) } +// resolveThreePartName applies resolveCatalog and resolveSchema to the catalog and +// schema of a "catalog.schema.name" UC identifier. A name that is not three parts, or +// that already contains a reference, is returned unchanged; a mix of references and +// literals is not supported. +func resolveThreePartName(b *bundle.Bundle, name string) string { + if dynvar.ContainsVariableReference(name) { + return name + } + parts, ok := splitUCName(name, 3) + if !ok { + return name + } + catalogName, schemaName := parts[0], parts[1] + return resolveCatalog(b, catalogName) + "." + resolveSchema(b, catalogName, schemaName) + "." + parts[2] +} + func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Resolve resources that depend on schemas before resolving schemas themselves. // The schema resolution below modifies schema.CatalogName, and findSchema @@ -175,32 +191,13 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia if idx == nil { continue } - // A name that already contains a reference is left as is; a mix of - // references and literals is not supported. - if dynvar.ContainsVariableReference(idx.Name) { - continue - } - // Name is a three-part "catalog.schema.index" UC identifier. - parts, ok := splitUCName(idx.Name, 3) - if !ok { - continue - } - catalogName, schemaName := parts[0], parts[1] - idx.Name = resolveCatalog(b, catalogName) + "." + resolveSchema(b, catalogName, schemaName) + "." + parts[2] + idx.Name = resolveThreePartName(b, idx.Name) } for _, f := range b.Config.Resources.Features { if f == nil { continue } - if dynvar.ContainsVariableReference(f.FullName) { - continue - } - parts, ok := splitUCName(f.FullName, 3) - if !ok { - continue - } - catalogName, schemaName := parts[0], parts[1] - f.FullName = resolveCatalog(b, catalogName) + "." + resolveSchema(b, catalogName, schemaName) + "." + parts[2] + f.FullName = resolveThreePartName(b, f.FullName) } for _, set := range b.Config.Resources.MaterializationSets { if set == nil { @@ -222,15 +219,7 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia if fs == nil { continue } - if dynvar.ContainsVariableReference(fs.Name) { - continue - } - parts, ok := splitUCName(fs.Name, 3) - if !ok { - continue - } - catalogName, schemaName := parts[0], parts[1] - fs.Name = resolveCatalog(b, catalogName) + "." + resolveSchema(b, catalogName, schemaName) + "." + parts[2] + fs.Name = resolveThreePartName(b, fs.Name) } for _, mse := range b.Config.Resources.ModelServingEndpoints { if mse == nil { diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index b835dd78948..f80fda21d14 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" "testing" + "time" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" @@ -22,6 +23,7 @@ import ( "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/common/types/duration" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/dashboards" @@ -153,15 +155,28 @@ var testConfig map[string]any = map[string]any{ }, }, + // The fake accepts a declarative feature alongside lookups and functions, which the real API + // rejects. "feature_specs": &resources.FeatureSpec{ Name: "main.default.my_feature_spec", - Features: []ml.Features{{ - FeatureLookup: &ml.FeatureLookup{ - Table: "main.default.my_table", - LookupKeys: []string{"id"}, - FeatureNames: []string{"amount"}, - }, - }}, + Features: []ml.Features{ + {FeatureLookup: &ml.FeatureLookup{ + Table: "main.default.my_table", + LookupKeys: []string{"id"}, + TimestampLookupKeys: []string{"ts"}, + LookbackWindow: duration.New(time.Hour), + FeatureNames: []string{"amount", "country"}, + RenameOutputs: []ml.RenameOutput{{Feature: "amount", Output: "total_amount"}}, + DefaultValues: []ml.DefaultValue{{Feature: "country", DefaultValue: "unknown"}}, + }}, + {FeatureFunction: &ml.FeatureFunction{ + Udf: "main.default.my_udf", + Output: "score", + InputBindings: []ml.FeatureFunctionInputBinding{{Parameter: "x", BoundFeature: "amount"}}, + }}, + {DeclarativeFeature: &ml.DeclarativeFeature{Feature: "main.default.my_feature"}}, + }, + ExcludeColumns: []string{"country"}, ExtraPipRequirements: &ml.PipRequirements{Requirements: []string{"numpy==2.1.0"}}, }, diff --git a/bundle/direct/dresources/feature_spec.go b/bundle/direct/dresources/feature_spec.go index 078e26e3b4b..48d071d1127 100644 --- a/bundle/direct/dresources/feature_spec.go +++ b/bundle/direct/dresources/feature_spec.go @@ -20,8 +20,8 @@ func (*ResourceFeatureSpec) New(client *databricks.WorkspaceClient) *ResourceFea } // PrepareState sorts the fields in FeatureSpec that are order-free lists, so a reorder in config is -// not a recreate. The FeatureSpec with the order-free list fields sorted is what gets sent and -// stored in the backend. +// not a recreate. The sorted copy is what is sent in the create request and persisted in state; the +// config is not modified. func (*ResourceFeatureSpec) PrepareState(input *resources.FeatureSpec) *ml.FeatureSpec { state := input.FeatureSpec sortOrderFreeLists(&state) @@ -33,8 +33,8 @@ func (r *ResourceFeatureSpec) DoRead(ctx context.Context, id string) (*ml.Featur if err != nil { return nil, err } - // Returns the FeatureSpec with its order-free list fields sorted, so a feature spec created - // outside the bundle also has them sorted. + // Sorted the same way as in PrepareState, so the two compare equal whatever order the API + // returns the lists in. sortOrderFreeLists(featureSpec) return featureSpec, nil } @@ -75,8 +75,8 @@ func featureFunctionInputBindingKey(x ml.FeatureFunctionInputBinding) (string, s } // sortOrderFreeLists sorts a FeatureSpec's ExcludeColumns and ExtraPipRequirements. Both are -// treated as sets by the server but stored as string lists, which cannot be compared by a key -// defined in KeyedSlices. Each field is replaced with the sorted list rather than sorted in place, +// treated as sets by the server but stored as string lists, so they are sorted on both sides, as +// normalizeAssignments in grants.go does for privileges. Each field is replaced with a sorted copy, // so the bundle config is not mutated. func sortOrderFreeLists(featureSpec *ml.FeatureSpec) { featureSpec.ExcludeColumns = sortedClone(featureSpec.ExcludeColumns) @@ -87,7 +87,8 @@ func sortOrderFreeLists(featureSpec *ml.FeatureSpec) { } } -// sortedClone keeps an empty list empty rather than nil, so the request body is unchanged. +// sortedClone returns a sorted copy of values. An empty list stays empty rather than nil, so the +// request body is unchanged. func sortedClone(values []string) []string { sorted := slices.Clone(values) slices.Sort(sorted) diff --git a/bundle/phases/messages.go b/bundle/phases/messages.go index 0e7a29a9c82..535d8086b59 100644 --- a/bundle/phases/messages.go +++ b/bundle/phases/messages.go @@ -63,8 +63,9 @@ or recreate the Feature View. Deleting or recreating a Feature View that a Featu not blocked, and leaves the FeatureSpec stale:` deleteOrRecreateFeatureSpecMessage = ` -This action will result in the deletion or recreation of the following feature specs. -Grants on them that are not declared in the bundle are lost:` +This action will result in the deletion or recreation of the following FeatureSpecs. +Feature Serving endpoints and training code that use a deleted FeatureSpec can no longer +load it, and a recreated one loses the grants the bundle does not declare:` ) // DataLossWarning is the warning shown when a non-interactive command is about @@ -119,6 +120,6 @@ The conversation history attached to them is permanently lost:` deleteFeatureMessage = `This action will result in the deletion of the following Feature Views. Deleting a Feature View that a FeatureSpec serves is not blocked, and leaves the FeatureSpec stale:` - deleteFeatureSpecMessage = `This action will result in the deletion of the following feature specs. + deleteFeatureSpecMessage = `This action will result in the deletion of the following FeatureSpecs. Feature Serving endpoints and training code that use them can no longer load them:` ) diff --git a/libs/testserver/feature_spec.go b/libs/testserver/feature_spec.go index 61670dbd0d2..db8b92c0599 100644 --- a/libs/testserver/feature_spec.go +++ b/libs/testserver/feature_spec.go @@ -26,12 +26,6 @@ func (s *FakeWorkspace) FeatureSpecsCreate(req Request) Response { // UC normalizes feature spec names to lowercase. featureSpec.Name = strings.ToLower(featureSpec.Name) - if len(strings.Split(featureSpec.Name, ".")) != 3 { - return Response{ - StatusCode: http.StatusBadRequest, - Body: fmt.Sprintf("name must be a three-part name, got %q", featureSpec.Name), - } - } if _, exists := s.FeatureSpecs[featureSpec.Name]; exists { return Response{ StatusCode: http.StatusConflict, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index a021fd8b53a..20aac63e2b7 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -11,10 +11,12 @@ import ( "mime/multipart" "net/http" "path" + "regexp" "strconv" "strings" "github.com/databricks/cli/libs/notebook" + "github.com/databricks/cli/libs/testserver/testsql" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -1177,10 +1179,19 @@ func AddDefaultHandlers(server *Server) { // error, custom status) can register its own handler for the same pattern // before calling AddDefaultHandlers, or stub it via test.toml in acceptance. server.Handle("POST", "/api/2.0/sql/statements", server.sqlExecuteStatement) + // The API also accepts a trailing slash, which acceptance scripts send. + server.Handle("POST", "/api/2.0/sql/statements/", server.sqlExecuteStatement) server.Handle("GET", "/api/2.0/sql/statements/{statement_id}", server.sqlGetStatement) server.Handle("GET", "/api/2.0/sql/statements/{statement_id}/result/chunks/{chunk_index}", server.sqlGetStatementResultChunk) server.Handle("POST", "/api/2.0/sql/statements/{statement_id}/cancel", server.sqlCancelStatement) + // The fake keeps no tables or functions, so the statements acceptance scripts + // run to set them up succeed without effect. + fixtureStatement := regexp.MustCompile(`(?i)^(CREATE\s+(TABLE|FUNCTION)|INSERT\s+INTO)\s`) + server.sqlHandler.HandleFallbackPattern(fixtureStatement, func(testsql.Request) testsql.Result { + return testsql.Result{} + }) + server.Handle("GET", "/api/2.0/preview/sql/data_sources", func(req Request) any { return req.Workspace.SqlDataSourcesList(req) }) diff --git a/libs/testserver/statements_test.go b/libs/testserver/statements_test.go index 5552f64f558..83059b3d5ef 100644 --- a/libs/testserver/statements_test.go +++ b/libs/testserver/statements_test.go @@ -1,7 +1,9 @@ package testserver import ( + "bytes" "encoding/json" + "net/http" "regexp" "testing" @@ -49,3 +51,56 @@ func TestHandleSQLPattern(t *testing.T) { require.NotNil(t, resp.Result) assert.Equal(t, [][]string{{"42"}}, resp.Result.DataArray) } + +// postSQL submits a statement over HTTP to path and returns the response. +func postSQL(t *testing.T, server *Server, path, statement string) *sql.StatementResponse { + t.Helper() + body, err := json.Marshal(sql.ExecuteStatementRequest{Statement: statement, WarehouseId: "wh-1", WaitTimeout: "10s"}) + require.NoError(t, err) + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, server.URL+path, bytes.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer token") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + var out sql.StatementResponse + require.NoError(t, json.NewDecoder(resp.Body).Decode(&out)) + return &out +} + +func TestDefaultHandlersAcceptFixtureStatements(t *testing.T) { + server := New(t) + AddDefaultHandlers(server) + + tests := []struct { + path string + statement string + want sql.StatementState + }{ + {"/api/2.0/sql/statements", "CREATE TABLE main.s.t (id INT)", sql.StatementStateSucceeded}, + {"/api/2.0/sql/statements/", "CREATE TABLE IF NOT EXISTS main.s.t AS SELECT 1", sql.StatementStateSucceeded}, + {"/api/2.0/sql/statements/", "create function main.s.f(x INT) RETURNS INT RETURN x", sql.StatementStateSucceeded}, + {"/api/2.0/sql/statements/", "INSERT INTO main.s.t VALUES (1)", sql.StatementStateSucceeded}, + {"/api/2.0/sql/statements/", "DROP TABLE main.s.t", sql.StatementStateFailed}, + } + for _, tc := range tests { + t.Run(tc.statement, func(t *testing.T) { + resp := postSQL(t, server, tc.path, tc.statement) + require.NotNil(t, resp.Status) + assert.Equal(t, tc.want, resp.Status.State) + }) + } +} + +func TestDefaultFixtureMatcherYieldsToTestMatcher(t *testing.T) { + server := New(t) + AddDefaultHandlers(server) + server.HandleSQL("CREATE TABLE main.s.t (id INT)", func(testsql.Request) testsql.Result { + return testsql.Result{Columns: []string{"a"}, Rows: [][]string{{"1"}}} + }) + + resp := postSQL(t, server, "/api/2.0/sql/statements", "CREATE TABLE main.s.t (id INT)") + require.NotNil(t, resp.Result) + assert.Equal(t, [][]string{{"1"}}, resp.Result.DataArray) +} diff --git a/libs/testserver/testsql/testsql.go b/libs/testserver/testsql/testsql.go index 0d28785f40f..2ecc0923d16 100644 --- a/libs/testserver/testsql/testsql.go +++ b/libs/testserver/testsql/testsql.go @@ -16,6 +16,7 @@ package testsql import ( "fmt" "regexp" + "slices" "strings" "sync" @@ -91,6 +92,7 @@ type statement struct { type Handler struct { mu sync.Mutex matchers []matcher + fallbacks []matcher statements map[string]*statement nextID int } @@ -123,12 +125,25 @@ func (h *Handler) Handle(statement string, fn func(Request) Result) { func (h *Handler) HandlePattern(re *regexp.Regexp, fn func(Request) Result) { h.mu.Lock() defer h.mu.Unlock() - h.matchers = append(h.matchers, matcher{ + h.matchers = append(h.matchers, patternMatcher(re, fn)) +} + +// HandleFallbackPattern registers a matcher like HandlePattern, but tried only +// when no Handle or HandlePattern matcher matches, whatever the registration +// order. +func (h *Handler) HandleFallbackPattern(re *regexp.Regexp, fn func(Request) Result) { + h.mu.Lock() + defer h.mu.Unlock() + h.fallbacks = append(h.fallbacks, patternMatcher(re, fn)) +} + +func patternMatcher(re *regexp.Regexp, fn func(Request) Result) matcher { + return matcher{ match: func(stmt string) []string { return re.FindStringSubmatch(stmt) }, fn: fn, - }) + } } // Submit matches the statement, runs the matcher once to capture its plan, @@ -215,7 +230,7 @@ func (h *Handler) plan(stmt string, params []sql.StatementParameterListItem) *st Code: sql.ServiceErrorCodeBadRequest, Message: "unsupported statement: " + stmt, }} - for _, m := range h.matchers { + for _, m := range slices.Concat(h.matchers, h.fallbacks) { groups := m.match(stmt) if groups == nil { continue diff --git a/libs/testserver/testsql/testsql_test.go b/libs/testserver/testsql/testsql_test.go index 7c70d038218..31228081396 100644 --- a/libs/testserver/testsql/testsql_test.go +++ b/libs/testserver/testsql/testsql_test.go @@ -55,6 +55,19 @@ func TestFirstMatchWins(t *testing.T) { assert.Equal(t, []string{"first"}, columnNames(resp)) } +func TestFallbackYieldsToMatchers(t *testing.T) { + e := testsql.New() + e.HandleFallbackPattern(regexp.MustCompile(`^CREATE`), func(testsql.Request) testsql.Result { + return testsql.Result{Columns: []string{"fallback"}} + }) + e.Handle("CREATE TABLE foo", func(testsql.Request) testsql.Result { + return testsql.Result{Columns: []string{"matcher"}} + }) + + assert.Equal(t, []string{"matcher"}, columnNames(e.Submit("CREATE TABLE foo", "10s", nil))) + assert.Equal(t, []string{"fallback"}, columnNames(e.Submit("CREATE TABLE bar", "10s", nil))) +} + func TestUnsupportedStatement(t *testing.T) { e := testsql.New() resp := e.Submit("SELECT 1", "10s", nil)