Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions acceptance/bundle/templates/lakeflow-integrations/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,12 @@ Validation OK!
>>> [CLI] bundle deploy -t dev
Building lakeflow_integrations_yaml...
Building python_artifact...
Uploading __generated__/my_lakeflow_integrations.examples.slack.send_slack_message.yml...
Uploading __generated__/my_lakeflow_integrations.examples.wait_for_run_sensor.WaitForRunSensor.yml...
Uploading __generated__/my_lakeflow_integrations.examples.call_rest_api.call_rest_api.yml...
Uploading __generated__/my_lakeflow_integrations.examples.create_jira_issue.create_jira_issue.yml...
Uploading __generated__/my_lakeflow_integrations.examples.send_slack_message.send_slack_message.yml...
Uploading __generated__/my_lakeflow_integrations.examples.wait_for_rest_api.WaitForRestApi.yml...
Uploading __generated__/my_lakeflow_integrations.examples.wait_for_slack_reaction.WaitForSlackReaction.yml...
Uploading dist/my_lakeflow_integrations-0.0.1-py3-none-any.whl...
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_lakeflow_integrations/dev/files...
Created jobs.send_slack_message
Created jobs.wait_for_run_sensor
Files: 11 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
Files: 12 uploaded, 0 deleted
Resources: 0 created, 0 changed, 0 deleted, 0 unchanged
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,29 @@

A Declarative Automation Bundles project that extends Lakeflow Jobs with custom integrations.

This bundle ships with runnable examples under `src/my_lakeflow_integrations/examples/` and
corresponding job definitions under `resources/`:
This bundle ships with examples under `src/my_lakeflow_integrations/examples/` that call external
services through Unity Catalog connections:

| Example | Type | What it does |
| --- | --- | --- |
| `slack.send_slack_message` | Function | Posts a message to Slack through a Unity Catalog connection. |
| `wait_for_run_sensor.WaitForRunSensor` | Sensor | Polls a job run until it reaches a terminal state. |
| `send_slack_message.send_slack_message` | Function | Posts a message to a Slack channel or direct message, with optional rich formatting. |
| `wait_for_slack_reaction.WaitForSlackReaction` | Sensor | Waits until someone reacts to a Slack message. |
| `create_jira_issue.create_jira_issue` | Function | Creates a Jira issue. |
| `call_rest_api.call_rest_api` | Function | Sends a request to a REST API and saves the response as task values. |
| `wait_for_rest_api.WaitForRestApi` | Sensor | Polls a REST API until its response meets a condition. |

Define jobs that run them under `resources/`.

## Getting started

1. Open this bundle in a Databricks workspace.
2. Click the **deployments rocket** 🚀 in the left sidebar to open the
**Deployments** panel, then click **Deploy**.
3. To run a deployed job, hover over the resource in the **Deployments** panel
and click **Run**.
3. To make the deployed integrations appear in the task palette, add them to `/Workspace/Users/<user>/.lakeflow_integrations.yml`. For example:
```yaml
integrations:
- '/Workspace/Users/<user>/.bundle/<bundle>/dev/artifacts/.internal/*.yml'
```

## Documentation

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ requires-python = ">=3.12,<3.13"
dependencies = [
# Any dependencies for jobs and pipelines in this project can be added here
# See also https://docs.databricks.com/dev-tools/bundles/library-dependencies
"databricks-lakeflow-integrations>=0.0.3,<0.1",
"databricks-lakeflow-integrations>=0.0.6,<0.1",
]

[tool.hatch.build.targets.wheel]
Expand All @@ -17,7 +17,6 @@ packages = [
[dependency-groups]
dev = [
# libraries part of serverless environment
"requests",
"databricks-sdk"
]

Expand Down

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
import json
from typing import Any, Optional
from urllib.parse import unquote, urlsplit

from databricks.lakeflow.integrations import (
HttpUcConnectionParameter,
integration,
ParameterMetadata,
TextAreaParameter,
)
from databricks.sdk import WorkspaceClient
from databricks.sdk.service.serving import ExternalFunctionRequestHttpMethod as Method

PREVIEW_LENGTH = 2000


@integration(
display_name="Call REST API",
description="Sends a request through a UC connection, saving the response as task values",
icon="globe",
promoted_parameter="path",
parameters=dict(
connection=ParameterMetadata(display_name="Connection", description="Unity Catalog connection to the API."),
path=ParameterMetadata(
display_name="Path", description="Path after the connection's base path. For example, jobs/42/refresh."
),
method=ParameterMetadata(display_name="Method", description="HTTP method of the request."),
query_params=ParameterMetadata(
display_name="Query parameters",
description="Added to the request URL. For example, limit with 10 adds ?limit=10.",
),
headers=ParameterMetadata(display_name="Headers", description="Extra headers to send with the request."),
body=ParameterMetadata(display_name="Body", description="Request body, in JSON."),
output_mappings=ParameterMetadata(
display_name="Output mappings",
description="Each key is a task value name and each value is its path in the response. For example, "
"job_id with id, or first_id with items[0].id.",
),
),
)
def call_rest_api(
connection: HttpUcConnectionParameter,
path: str,
method: Method = Method.GET,
query_params: Optional[dict[str, str]] = None,
headers: Optional[dict[str, str]] = None,
body: Optional[TextAreaParameter] = None,
output_mappings: Optional[dict[str, str]] = None,
) -> None:
if ".." in unquote(urlsplit(path).path).split("/"):
raise ValueError(f"path can't contain '..' segments, got: {path}")
json_body = parse_json_body(body)

workspace = WorkspaceClient()
print(f"Sending {method.value} {path}")
response = workspace.serving_endpoints.http_request(
conn=connection, method=method, path=path, params=query_params, headers=headers, json=json_body
)
print(f"Response: {preview(response.text)}")

outputs = extract_outputs(parse_response(response), output_mappings or {})
for name, value in outputs.items():
workspace.dbutils.jobs.taskValues.set(key=name, value=value)
print(f"Saved task value {name}: {preview(json.dumps(value))}")


def parse_json_body(body: Optional[str]) -> Any:
try:
return json.loads(body) if body else None
except json.JSONDecodeError as error:
raise ValueError(f"body must be valid JSON: {error}") from None


def parse_response(response: Any) -> Any:
try:
return response.json()
except ValueError:
return response.text


def preview(text: str) -> str:
if not text:
return "(empty)"
if len(text) > PREVIEW_LENGTH:
return f"{text[:PREVIEW_LENGTH]}... (truncated)"
return text


def extract_outputs(response: Any, mappings: dict[str, str]) -> dict[str, Any]:
outputs = {}
for name, response_path in mappings.items():
value = extract_nested_value(response, response_path)
if value is None:
raise RuntimeError(f"The response has no value at {response_path}")

outputs[name] = value
return outputs


def extract_nested_value(response: Any, response_path: str) -> Any:
# result.items[0].id becomes the keys result, items, 0, and id. A leading [0] indexes a list response.
normalized_path = response_path.replace("[", ".").replace("]", "").lstrip(".")
value = response
for key in normalized_path.split("."):
if isinstance(value, dict):
value = value.get(key)
elif isinstance(value, list) and key.isdigit() and int(key) < len(value):
value = value[int(key)]
else:
return None
return value
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
import json
from typing import Annotated, Any, Optional

from databricks.lakeflow.integrations import (
HttpConnectionFilter,
HttpUcConnectionParameter,
integration,
ParameterMetadata,
TextAreaParameter,
)
from databricks.sdk import WorkspaceClient
from databricks.sdk.service.serving import ExternalFunctionRequestHttpMethod

JiraConnection = Annotated[HttpUcConnectionParameter, HttpConnectionFilter(host="https://api.atlassian.com")]


@integration(
display_name="Create Jira issue",
description="Creates a Jira issue",
icon="plus-circle",
promoted_parameter="project_key",
parameters=dict(
connection=ParameterMetadata(
display_name="Connection", description="Unity Catalog connection to the Jira Cloud REST API."
),
project_key=ParameterMetadata(display_name="Project key", description="For example, DATA."),
summary=ParameterMetadata(display_name="Summary", description="Title of the issue."),
issue_type=ParameterMetadata(display_name="Issue type", description="For example, Bug."),
description=ParameterMetadata(display_name="Description", description="Body of the issue, in plain text."),
assignee_email=ParameterMetadata(display_name="Assignee email", description="For example, ana@example.com."),
labels=ParameterMetadata(display_name="Labels", description="For example, pipeline."),
priority=ParameterMetadata(display_name="Priority", description="For example, High."),
fields=ParameterMetadata(
display_name="Other fields",
description="More Jira fields to set, as a JSON object of field IDs and values. "
'For example, {"customfield_10000": "data-eng"}. The fields above take precedence.',
),
),
)
def create_jira_issue(
connection: JiraConnection,
project_key: str,
summary: str,
issue_type: str = "Task",
description: Optional[TextAreaParameter] = None,
assignee_email: Optional[str] = None,
labels: Optional[list[str]] = None,
priority: Optional[str] = None,
fields: Optional[TextAreaParameter] = None,
) -> None:
workspace = WorkspaceClient()
assignee_id = find_account_id_by_email(workspace, connection, assignee_email) if assignee_email else None
optional_fields = {
"description": plain_text_to_jira_document(description) if description else None,
"assignee": {"accountId": assignee_id} if assignee_id else None,
"labels": labels,
"priority": {"name": priority} if priority else None,
}
issue_fields: dict[str, Any] = {
**parse_other_fields(fields),
"project": {"key": project_key},
"summary": summary,
"issuetype": {"name": issue_type},
**{key: value for key, value in optional_fields.items() if value},
}

result = send_jira_request(
workspace, connection, ExternalFunctionRequestHttpMethod.POST, "issue", {"fields": issue_fields}
)
print(f"Created issue {result['key']}")
workspace.dbutils.jobs.taskValues.set(key="issue_key", value=result["key"])


def find_account_id_by_email(workspace: WorkspaceClient, connection: str, email: str) -> str:
users = send_jira_request(
workspace, connection, ExternalFunctionRequestHttpMethod.GET, "user/search", None, params={"query": email}
)
# Jira searches by prefix and returns a null email for users with a private profile.
matches = [user for user in users if (user.get("emailAddress") or "").lower() == email.lower()]
if len(matches) != 1:
raise ValueError(f"Found {len(matches)} Jira users with email {email}. Enter the email of exactly one user.")
return matches[0]["accountId"]


def parse_other_fields(fields: Optional[str]) -> dict[str, Any]:
try:
parsed = json.loads(fields) if fields else {}
except json.JSONDecodeError as error:
raise ValueError(f"Other fields must be a JSON object: {error}") from None
if not isinstance(parsed, dict):
raise ValueError("Other fields must be a JSON object, not a list or a single value")
return parsed


def send_jira_request(
workspace: WorkspaceClient,
connection: str,
method: ExternalFunctionRequestHttpMethod,
path: str,
body: Optional[dict[str, Any]],
params: Optional[dict[str, str]] = None,
) -> Any:
response = workspace.serving_endpoints.http_request(
conn=connection, method=method, path=path, json=body, params=params
)
try:
return response.json()
except ValueError:
raise RuntimeError(f"Jira returned a non-JSON response: {response.text}") from None


def plain_text_to_jira_document(text: str) -> dict[str, Any]:
# Jira's v3 API only takes rich text as an Atlassian Document Format document.
paragraphs = [
{"type": "paragraph", "content": [{"type": "text", "text": line}]} for line in text.splitlines() if line.strip()
]
return {"type": "doc", "version": 1, "content": paragraphs}
Loading
Loading