Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 14 additions & 37 deletions acceptance/bin/create_feature_spec_fixtures.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,44 +6,21 @@
Usage: create_feature_spec_fixtures.py CATALOG.SCHEMA
"""

import json
import os
import subprocess
import sys
import time

schema_full_name = sys.argv[1]
cli = os.environ.get("CLI", "databricks")
warehouse_id = os.environ["TEST_DEFAULT_WAREHOUSE_ID"]

statements = [
f"CREATE TABLE {schema_full_name}.user_features "
"(user_id INT NOT NULL, age INT, country STRING, CONSTRAINT user_features_pk PRIMARY KEY (user_id))",
f"CREATE TABLE {schema_full_name}.tenant_user_features "
"(tenant_id INT NOT NULL, user_id INT NOT NULL, age INT, country STRING, "
"CONSTRAINT tenant_user_features_pk PRIMARY KEY (tenant_id, user_id))",
f"CREATE FUNCTION {schema_full_name}.score(x INT) RETURNS DOUBLE LANGUAGE PYTHON AS $$ return float(x) $$",
f"CREATE FUNCTION {schema_full_name}.country_score(x INT, y STRING) RETURNS DOUBLE LANGUAGE PYTHON "
"AS $$ return float(x) + len(y) $$",
]

from util import run_statements

def call_api(*args):
result = subprocess.run([cli, "api", *args], capture_output=True, text=True)
if result.returncode != 0:
print(f"databricks api {' '.join(args)} failed: {result.stderr}", file=sys.stderr)
sys.exit(1)
return json.loads(result.stdout)

schema_full_name = sys.argv[1]

for statement in statements:
payload = json.dumps({"warehouse_id": warehouse_id, "statement": statement, "wait_timeout": "50s"})
response = call_api("post", "/api/2.0/sql/statements/", "--json", payload)
# A warehouse that is still starting outlasts wait_timeout.
deadline = time.monotonic() + 600
while response["status"]["state"] in ("PENDING", "RUNNING") and time.monotonic() < deadline:
time.sleep(5)
response = call_api("get", f"/api/2.0/sql/statements/{response['statement_id']}")
if response["status"]["state"] != "SUCCEEDED":
print(f"Statement {statement!r} ended in {response['status']}", file=sys.stderr)
sys.exit(1)
run_statements(
[
f"CREATE TABLE {schema_full_name}.user_features "
"(user_id INT NOT NULL, age INT, country STRING, CONSTRAINT user_features_pk PRIMARY KEY (user_id))",
f"CREATE TABLE {schema_full_name}.tenant_user_features "
"(tenant_id INT NOT NULL, user_id INT NOT NULL, age INT, country STRING, "
"CONSTRAINT tenant_user_features_pk PRIMARY KEY (tenant_id, user_id))",
f"CREATE FUNCTION {schema_full_name}.score(x INT) RETURNS DOUBLE LANGUAGE PYTHON AS $$ return float(x) $$",
f"CREATE FUNCTION {schema_full_name}.country_score(x INT, y STRING) RETURNS DOUBLE LANGUAGE PYTHON "
"AS $$ return float(x) + len(y) $$",
]
)
31 changes: 8 additions & 23 deletions acceptance/bin/create_table.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
import sys
import time

from util import run_statements

table_name = sys.argv[1]
# Extract catalog.schema from table_name
parts = table_name.split(".")
Expand All @@ -24,36 +26,19 @@ def run_cli(*args):
return result


def execute_sql(warehouse_id, sql):
"""Execute SQL using the API endpoint."""
payload = json.dumps({"warehouse_id": warehouse_id, "statement": sql, "wait_timeout": "30s"})
return run_cli("api", "post", "/api/2.0/sql/statements/", "--json", payload)


# Get warehouse ID from environment variable
warehouse_id = os.environ["TEST_DEFAULT_WAREHOUSE_ID"]

# Create a simple table
sql = f"CREATE TABLE IF NOT EXISTS {table_name} (id INT, value STRING, timestamp TIMESTAMP) USING DELTA"

result = execute_sql(warehouse_id, sql)
if result.returncode != 0:
print(f"Failed to create table: {result.stderr}", file=sys.stderr)
sys.exit(1)

run_statements([f"CREATE TABLE IF NOT EXISTS {table_name} (id INT, value STRING, timestamp TIMESTAMP) USING DELTA"])
print(f"Created table {table_name}")

# Insert some sample data so the monitor has something to analyze
insert_sql = f"""INSERT INTO {table_name} VALUES
run_statements(
[
f"""INSERT INTO {table_name} VALUES
(1, 'test1', current_timestamp()),
(2, 'test2', current_timestamp()),
(3, 'test3', current_timestamp())"""

result = execute_sql(warehouse_id, insert_sql)
if result.returncode != 0:
print(f"Failed to insert data: {result.stderr}", file=sys.stderr)
sys.exit(1)

]
)
print(f"Inserted sample data into {table_name}")

# Wait for table to be visible in Unity Catalog
Expand Down
18 changes: 18 additions & 0 deletions acceptance/bin/util.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import json
import os
import shlex
import subprocess
import sys
import time

VERBOSE = False

Expand Down Expand Up @@ -31,3 +33,19 @@ def run(cmd):
if result.returncode != 0:
raise RunError(f"{cmd} failed with code {result.returncode}")
return result


def run_statements(statements):
"""Runs SQL statements on the test warehouse one after another, waiting for each to succeed."""
cli = os.environ.get("CLI", "databricks")
warehouse_id = os.environ["TEST_DEFAULT_WAREHOUSE_ID"]
for statement in statements:
payload = json.dumps({"warehouse_id": warehouse_id, "statement": statement, "wait_timeout": "50s"})
response = run_json([cli, "api", "post", "/api/2.0/sql/statements/", "--json", payload])
# A warehouse that is still starting outlasts wait_timeout.
deadline = time.monotonic() + 600
while response["status"]["state"] in ("PENDING", "RUNNING") and time.monotonic() < deadline:
time.sleep(5)
response = run_json([cli, "api", "get", f"/api/2.0/sql/statements/{response['statement_id']}"])
if response["status"]["state"] != "SUCCEEDED":
raise RunError(f"Statement {statement!r} ended in {response['status']}")
5 changes: 2 additions & 3 deletions acceptance/bundle/deployment/bind/feature_spec/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

=== Create the feature spec outside the bundle

>>> [CLI] api post /api/2.0/feature-engineering/feature-specs --json {
>>> [CLI] feature-engineering create-feature-spec --json {
"name": "main.fspec_[UNIQUE_NAME].boundspec",
"features": [{"feature_lookup": {"table": "main.fspec_[UNIQUE_NAME].user_features", "lookup_keys": ["user_id"], "feature_names": ["age"]}}]
}
Expand Down Expand Up @@ -51,5 +51,4 @@ Resources:
Name: main.fspec_[UNIQUE_NAME].boundspec
URL: (not deployed)

>>> [CLI] api delete /api/2.0/feature-engineering/feature-specs/main.fspec_[UNIQUE_NAME].boundspec
{}
>>> [CLI] feature-engineering delete-feature-spec main.fspec_[UNIQUE_NAME].boundspec
17 changes: 3 additions & 14 deletions acceptance/bundle/deployment/bind/feature_spec/script
Original file line number Diff line number Diff line change
@@ -1,18 +1,7 @@
# Creating a feature spec resolves its lookup tables in Unity Catalog, so the test creates them in
# a schema of its own. Dropping the schema also drops the feature spec.
export SCHEMA_FULL_NAME="main.fspec_$UNIQUE_NAME"
trace $CLI schemas create "fspec_$UNIQUE_NAME" main -o json | jq '{full_name}'
trace create_feature_spec_fixtures.py "$SCHEMA_FULL_NAME"

cleanup() {
$CLI schemas delete "$SCHEMA_FULL_NAME" --force >> LOG.cleanup 2>&1 || true
}
trap cleanup EXIT

envsubst < databricks.yml.tmpl > databricks.yml
feature_spec_setup

title "Create the feature spec outside the bundle"
MSYS_NO_PATHCONV=1 trace $CLI api post /api/2.0/feature-engineering/feature-specs --json "{
trace $CLI feature-engineering create-feature-spec --json "{
\"name\": \"$SCHEMA_FULL_NAME.boundspec\",
\"features\": [{\"feature_lookup\": {\"table\": \"$SCHEMA_FULL_NAME.user_features\", \"lookup_keys\": [\"user_id\"], \"feature_names\": [\"age\"]}}]
}" | jq '{name}'
Expand All @@ -26,4 +15,4 @@ trace $CLI bundle plan
trace $CLI bundle deployment unbind spec1
trace $CLI bundle summary

MSYS_NO_PATHCONV=1 trace $CLI api delete /api/2.0/feature-engineering/feature-specs/$SCHEMA_FULL_NAME.boundspec
trace $CLI feature-engineering delete-feature-spec $SCHEMA_FULL_NAME.boundspec
5 changes: 0 additions & 5 deletions acceptance/bundle/deployment/bind/feature_spec/test.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,6 @@ Ignore = [
".databricks",
]

# Fake SQL endpoint so create_feature_spec_fixtures.py works against the local test server too.
[[Server]]
Pattern = "POST /api/2.0/sql/statements/"
Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}'

# Azure and GCP hosts carry the workspace ID, so their summary URLs have no ?w=<id>.
[[Repls]]
Old = '\?[ow]=\[WORKSPACE_ID\]'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,4 @@

# Creating a feature spec resolves its lookup table in Unity Catalog, and the invariant harness
# provisions no tables of its own.
echo "Creating temporary schema and feature table: main.fspec_inv_$UNIQUE_NAME"

$CLI schemas create fspec_inv_$UNIQUE_NAME main -o json | jq '{full_name}'
create_feature_spec_fixtures.py main.fspec_inv_$UNIQUE_NAME
feature_spec_fixtures "fspec_inv_$UNIQUE_NAME"
5 changes: 0 additions & 5 deletions acceptance/bundle/invariant/test.toml
Original file line number Diff line number Diff line change
Expand Up @@ -128,11 +128,6 @@ EnvMatrixExclude.no_data_quality_monitor_on_cloud = ["CONFIG_Cloud=true", "INPUT
EnvMatrixExclude.no_feature_spec_dms = ["DMS=true", "INPUT_CONFIG=feature_spec.yml.tmpl"]
# Same for the "postgres_data_apis." prefix.
EnvMatrixExclude.no_postgres_data_api_dms = ["DMS=true", "INPUT_CONFIG=postgres_data_api.yml.tmpl"]
# Fake SQL endpoint for local tests

[[Server]]
Pattern = "POST /api/2.0/sql/statements/"
Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}'

[[Server]]
Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,3 @@ resources:
table: main.default.user_features
lookup_keys: [user_id]
feature_names: [age]
grants:
- principal: account users
privileges: [EXECUTE]
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
Cloud = true
Cloud = false
EnvMatrix.DMS = ["", "true"]
EnvMatrix.PYDAB_VERSION = ["current"]
8 changes: 0 additions & 8 deletions acceptance/bundle/python/feature_specs-support/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,6 @@
}
}
],
"grants": [
{
"principal": "account users",
"privileges": [
"EXECUTE"
]
}
],
"name": "main.default.my_feature_spec_1_updated"
},
"my_feature_spec_2": {
Expand Down
2 changes: 1 addition & 1 deletion acceptance/bundle/python/feature_specs-support/test.toml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
Cloud = true
Cloud = false # tests don't interact with APIs

# feature specs are only supported in the current version of the wheel
EnvMatrix.PYDAB_VERSION = ["current"]
9 changes: 5 additions & 4 deletions acceptance/bundle/resources/feature_specs/basic/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
>>> [CLI] bundle summary -o json
{
"id": "[SPEC1_ID]",
"url": "[DATABRICKS_URL]/explore/data/functions/main/fspec_[UNIQUE_NAME]/myspec?w=[WORKSPACE_ID]"
"url": "[DATABRICKS_URL]/explore/data/functions/main/fspec_[UNIQUE_NAME]/myspec"
}

=== A second plan on an unchanged bundle must report no changes
Expand All @@ -116,8 +116,9 @@ Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged
>>> [CLI] bundle deploy --auto-approve
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-feature-specs-basic-[UNIQUE_NAME]/default/files...

This action will result in the deletion or recreation of the following feature specs.
Grants on them that are not declared in the bundle are lost:
This action will result in the deletion or recreation of the following FeatureSpecs.
Feature Serving endpoints and training code that use a deleted FeatureSpec can no longer
load it, and a recreated one loses the grants the bundle does not declare:
recreate resources.feature_specs.spec1
Recreated feature_specs.spec1
Files: 1 uploaded, 0 deleted
Expand Down Expand Up @@ -170,7 +171,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged
The following resources will be deleted:
delete resources.feature_specs.spec1

This action will result in the deletion of the following feature specs.
This action will result in the deletion of the following FeatureSpecs.
Feature Serving endpoints and training code that use them can no longer load them:
delete resources.feature_specs.spec1

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged
The following resources will be deleted:
delete resources.feature_specs.spec1

This action will result in the deletion of the following feature specs.
This action will result in the deletion of the following FeatureSpecs.
Feature Serving endpoints and training code that use them can no longer load them:
delete resources.feature_specs.spec1

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

=== Delete the feature spec out of band

>>> [CLI] api delete /api/2.0/feature-engineering/feature-specs/main.fspec_[UNIQUE_NAME].myspec
{}
>>> [CLI] feature-engineering delete-feature-spec main.fspec_[UNIQUE_NAME].myspec

=== Plan should detect the feature spec is gone and re-create it

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ feature_spec_setup
trace $CLI bundle deploy

title "Delete the feature spec out of band"
MSYS_NO_PATHCONV=1 trace $CLI api delete /api/2.0/feature-engineering/feature-specs/$SCHEMA_FULL_NAME.myspec
trace $CLI feature-engineering delete-feature-spec $SCHEMA_FULL_NAME.myspec

title "Plan should detect the feature spec is gone and re-create it"
trace $CLI bundle plan
Expand Down
9 changes: 5 additions & 4 deletions acceptance/bundle/resources/feature_specs/test.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ Cloud = true
# mirrors Universe.
EnvMatrixExclude.no_feature_spec_dms = ["DMS=true"]

# Fake SQL endpoint so create_feature_spec_fixtures.py works against the local test server too.
[[Server]]
Pattern = "POST /api/2.0/sql/statements/"
Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}'
# A summary URL has ?w=<id> only when the host does not already carry the workspace ID.
[[Repls]]
Old = '\?[ow]=\[WORKSPACE_ID\]'
New = ''
Order = 1000
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,9 @@ Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged
>>> [CLI] bundle deploy --auto-approve
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-feature-specs-uppercase-name-[UNIQUE_NAME]/default/files...

This action will result in the deletion or recreation of the following feature specs.
Grants on them that are not declared in the bundle are lost:
This action will result in the deletion or recreation of the following FeatureSpecs.
Feature Serving endpoints and training code that use a deleted FeatureSpec can no longer
load it, and a recreated one loses the grants the bundle does not declare:
recreate resources.feature_specs.spec1
Recreated feature_specs.spec1
Files: 1 uploaded, 0 deleted
Expand Down Expand Up @@ -106,7 +107,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged
The following resources will be deleted:
delete resources.feature_specs.spec1

This action will result in the deletion of the following feature specs.
This action will result in the deletion of the following FeatureSpecs.
Feature Serving endpoints and training code that use them can no longer load them:
delete resources.feature_specs.spec1

Expand Down
5 changes: 0 additions & 5 deletions acceptance/bundle/resources/features/lifecycle/test.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,3 @@ Ignore = [
".databricks",
"databricks.yml",
]

# Fake SQL endpoint so create_table.py works against the local test server too.
[[Server]]
Pattern = "POST /api/2.0/sql/statements/"
Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}'
7 changes: 4 additions & 3 deletions acceptance/bundle/resources/grants/feature_specs/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
>>> [CLI] bundle deploy --auto-approve
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-feature-spec-grants-[UNIQUE_NAME]/default/files...

This action will result in the deletion or recreation of the following feature specs.
Grants on them that are not declared in the bundle are lost:
This action will result in the deletion or recreation of the following FeatureSpecs.
Feature Serving endpoints and training code that use a deleted FeatureSpec can no longer
load it, and a recreated one loses the grants the bundle does not declare:
recreate resources.feature_specs.my_feature_spec
Recreated feature_specs.my_feature_spec
Updated feature_specs.my_feature_spec.grants
Expand All @@ -66,7 +67,7 @@ Resources: 1 created, 1 changed, 1 deleted, 0 unchanged
The following resources will be deleted:
delete resources.feature_specs.my_feature_spec

This action will result in the deletion of the following feature specs.
This action will result in the deletion of the following FeatureSpecs.
Feature Serving endpoints and training code that use them can no longer load them:
delete resources.feature_specs.my_feature_spec

Expand Down
Loading
Loading