diff --git a/acceptance/bundle/templates/lakeflow-integrations/output.txt b/acceptance/bundle/templates/lakeflow-integrations/output.txt index 2a4e773c907..91176baa582 100644 --- a/acceptance/bundle/templates/lakeflow-integrations/output.txt +++ b/acceptance/bundle/templates/lakeflow-integrations/output.txt @@ -30,11 +30,12 @@ Validation OK! >>> [CLI] bundle deploy -t dev Building lakeflow_integrations_yaml... Building python_artifact... -Uploading __generated__/my_lakeflow_integrations.examples.slack.send_slack_message.yml... -Uploading __generated__/my_lakeflow_integrations.examples.wait_for_run_sensor.WaitForRunSensor.yml... +Uploading __generated__/my_lakeflow_integrations.examples.call_rest_api.call_rest_api.yml... +Uploading __generated__/my_lakeflow_integrations.examples.create_jira_issue.create_jira_issue.yml... +Uploading __generated__/my_lakeflow_integrations.examples.send_slack_message.send_slack_message.yml... +Uploading __generated__/my_lakeflow_integrations.examples.wait_for_rest_api.WaitForRestApi.yml... +Uploading __generated__/my_lakeflow_integrations.examples.wait_for_slack_reaction.WaitForSlackReaction.yml... Uploading dist/my_lakeflow_integrations-0.0.1-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_lakeflow_integrations/dev/files... -Created jobs.send_slack_message -Created jobs.wait_for_run_sensor -Files: 11 uploaded, 0 deleted -Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +Files: 12 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 0 unchanged diff --git a/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/README.md b/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/README.md index 671ad41f535..4a4345a25d3 100644 --- a/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/README.md +++ b/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/README.md @@ -2,21 +2,29 @@ A Declarative Automation Bundles project that extends Lakeflow Jobs with custom integrations. -This bundle ships with runnable examples under `src/my_lakeflow_integrations/examples/` and -corresponding job definitions under `resources/`: +This bundle ships with examples under `src/my_lakeflow_integrations/examples/` that call external +services through Unity Catalog connections: | Example | Type | What it does | | --- | --- | --- | -| `slack.send_slack_message` | Function | Posts a message to Slack through a Unity Catalog connection. | -| `wait_for_run_sensor.WaitForRunSensor` | Sensor | Polls a job run until it reaches a terminal state. | +| `send_slack_message.send_slack_message` | Function | Posts a message to a Slack channel or direct message, with optional rich formatting. | +| `wait_for_slack_reaction.WaitForSlackReaction` | Sensor | Waits until someone reacts to a Slack message. | +| `create_jira_issue.create_jira_issue` | Function | Creates a Jira issue. | +| `call_rest_api.call_rest_api` | Function | Sends a request to a REST API and saves the response as task values. | +| `wait_for_rest_api.WaitForRestApi` | Sensor | Polls a REST API until its response meets a condition. | + +Define jobs that run them under `resources/`. ## Getting started 1. Open this bundle in a Databricks workspace. 2. Click the **deployments rocket** 🚀 in the left sidebar to open the **Deployments** panel, then click **Deploy**. -3. To run a deployed job, hover over the resource in the **Deployments** panel - and click **Run**. +3. To make the deployed integrations appear in the task palette, add them to `/Workspace/Users//.lakeflow_integrations.yml`. For example: + ```yaml + integrations: + - '/Workspace/Users//.bundle//dev/artifacts/.internal/*.yml' + ``` ## Documentation diff --git a/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/pyproject.toml b/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/pyproject.toml index ad46ab6ebd4..52895555c95 100644 --- a/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/pyproject.toml +++ b/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/pyproject.toml @@ -6,7 +6,7 @@ requires-python = ">=3.12,<3.13" dependencies = [ # Any dependencies for jobs and pipelines in this project can be added here # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies - "databricks-lakeflow-integrations>=0.0.3,<0.1", + "databricks-lakeflow-integrations>=0.0.6,<0.1", ] [tool.hatch.build.targets.wheel] @@ -17,7 +17,6 @@ packages = [ [dependency-groups] dev = [ # libraries part of serverless environment - "requests", "databricks-sdk" ] diff --git a/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/resources/send_slack_message.job.yml b/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/resources/send_slack_message.job.yml deleted file mode 100644 index bb64af9d278..00000000000 --- a/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/resources/send_slack_message.job.yml +++ /dev/null @@ -1,24 +0,0 @@ -resources: - jobs: - send_slack_message: - name: "[example] send_slack_message" - - tasks: - - task_key: slack - environment_key: my_lakeflow_integrations_environment - python_operator_task: - main: my_lakeflow_integrations.examples.slack.send_slack_message - parameters: - - name: "conn_id" - value: "CHANGEME" - - name: "text" - value: "Hello, World" - - name: "channel" - value: "CHANGEME" - - environments: - - environment_key: my_lakeflow_integrations_environment - spec: - environment_version: "5" - dependencies: - - ../dist/*.whl diff --git a/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/resources/wait_for_run_sensor.job.yml b/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/resources/wait_for_run_sensor.job.yml deleted file mode 100644 index ed340fb7e92..00000000000 --- a/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/resources/wait_for_run_sensor.job.yml +++ /dev/null @@ -1,20 +0,0 @@ -resources: - jobs: - wait_for_run_sensor: - name: "[example] wait_for_run_sensor" - - tasks: - - task_key: wait_for_run_sensor - environment_key: my_lakeflow_integrations_environment - python_operator_task: - main: my_lakeflow_integrations.examples.wait_for_run_sensor.WaitForRunSensor - parameters: - - name: "run_id" - value: "CHANGEME" - - environments: - - environment_key: my_lakeflow_integrations_environment - spec: - environment_version: "5" - dependencies: - - ../dist/*.whl diff --git a/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/call_rest_api.py b/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/call_rest_api.py new file mode 100644 index 00000000000..3f678a8662c --- /dev/null +++ b/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/call_rest_api.py @@ -0,0 +1,111 @@ +import json +from typing import Any, Optional +from urllib.parse import unquote, urlsplit + +from databricks.lakeflow.integrations import ( + HttpUcConnectionParameter, + integration, + ParameterMetadata, + TextAreaParameter, +) +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.serving import ExternalFunctionRequestHttpMethod as Method + +PREVIEW_LENGTH = 2000 + + +@integration( + display_name="Call REST API", + description="Sends a request through a UC connection, saving the response as task values", + icon="globe", + promoted_parameter="path", + parameters=dict( + connection=ParameterMetadata(display_name="Connection", description="Unity Catalog connection to the API."), + path=ParameterMetadata( + display_name="Path", description="Path after the connection's base path. For example, jobs/42/refresh." + ), + method=ParameterMetadata(display_name="Method", description="HTTP method of the request."), + query_params=ParameterMetadata( + display_name="Query parameters", + description="Added to the request URL. For example, limit with 10 adds ?limit=10.", + ), + headers=ParameterMetadata(display_name="Headers", description="Extra headers to send with the request."), + body=ParameterMetadata(display_name="Body", description="Request body, in JSON."), + output_mappings=ParameterMetadata( + display_name="Output mappings", + description="Each key is a task value name and each value is its path in the response. For example, " + "job_id with id, or first_id with items[0].id.", + ), + ), +) +def call_rest_api( + connection: HttpUcConnectionParameter, + path: str, + method: Method = Method.GET, + query_params: Optional[dict[str, str]] = None, + headers: Optional[dict[str, str]] = None, + body: Optional[TextAreaParameter] = None, + output_mappings: Optional[dict[str, str]] = None, +) -> None: + if ".." in unquote(urlsplit(path).path).split("/"): + raise ValueError(f"path can't contain '..' segments, got: {path}") + json_body = parse_json_body(body) + + workspace = WorkspaceClient() + print(f"Sending {method.value} {path}") + response = workspace.serving_endpoints.http_request( + conn=connection, method=method, path=path, params=query_params, headers=headers, json=json_body + ) + print(f"Response: {preview(response.text)}") + + outputs = extract_outputs(parse_response(response), output_mappings or {}) + for name, value in outputs.items(): + workspace.dbutils.jobs.taskValues.set(key=name, value=value) + print(f"Saved task value {name}: {preview(json.dumps(value))}") + + +def parse_json_body(body: Optional[str]) -> Any: + try: + return json.loads(body) if body else None + except json.JSONDecodeError as error: + raise ValueError(f"body must be valid JSON: {error}") from None + + +def parse_response(response: Any) -> Any: + try: + return response.json() + except ValueError: + return response.text + + +def preview(text: str) -> str: + if not text: + return "(empty)" + if len(text) > PREVIEW_LENGTH: + return f"{text[:PREVIEW_LENGTH]}... (truncated)" + return text + + +def extract_outputs(response: Any, mappings: dict[str, str]) -> dict[str, Any]: + outputs = {} + for name, response_path in mappings.items(): + value = extract_nested_value(response, response_path) + if value is None: + raise RuntimeError(f"The response has no value at {response_path}") + + outputs[name] = value + return outputs + + +def extract_nested_value(response: Any, response_path: str) -> Any: + # result.items[0].id becomes the keys result, items, 0, and id. A leading [0] indexes a list response. + normalized_path = response_path.replace("[", ".").replace("]", "").lstrip(".") + value = response + for key in normalized_path.split("."): + if isinstance(value, dict): + value = value.get(key) + elif isinstance(value, list) and key.isdigit() and int(key) < len(value): + value = value[int(key)] + else: + return None + return value diff --git a/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/create_jira_issue.py b/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/create_jira_issue.py new file mode 100644 index 00000000000..91db7bf7fd0 --- /dev/null +++ b/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/create_jira_issue.py @@ -0,0 +1,117 @@ +import json +from typing import Annotated, Any, Optional + +from databricks.lakeflow.integrations import ( + HttpConnectionFilter, + HttpUcConnectionParameter, + integration, + ParameterMetadata, + TextAreaParameter, +) +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.serving import ExternalFunctionRequestHttpMethod + +JiraConnection = Annotated[HttpUcConnectionParameter, HttpConnectionFilter(host="https://api.atlassian.com")] + + +@integration( + display_name="Create Jira issue", + description="Creates a Jira issue", + icon="plus-circle", + promoted_parameter="project_key", + parameters=dict( + connection=ParameterMetadata( + display_name="Connection", description="Unity Catalog connection to the Jira Cloud REST API." + ), + project_key=ParameterMetadata(display_name="Project key", description="For example, DATA."), + summary=ParameterMetadata(display_name="Summary", description="Title of the issue."), + issue_type=ParameterMetadata(display_name="Issue type", description="For example, Bug."), + description=ParameterMetadata(display_name="Description", description="Body of the issue, in plain text."), + assignee_email=ParameterMetadata(display_name="Assignee email", description="For example, ana@example.com."), + labels=ParameterMetadata(display_name="Labels", description="For example, pipeline."), + priority=ParameterMetadata(display_name="Priority", description="For example, High."), + fields=ParameterMetadata( + display_name="Other fields", + description="More Jira fields to set, as a JSON object of field IDs and values. " + 'For example, {"customfield_10000": "data-eng"}. The fields above take precedence.', + ), + ), +) +def create_jira_issue( + connection: JiraConnection, + project_key: str, + summary: str, + issue_type: str = "Task", + description: Optional[TextAreaParameter] = None, + assignee_email: Optional[str] = None, + labels: Optional[list[str]] = None, + priority: Optional[str] = None, + fields: Optional[TextAreaParameter] = None, +) -> None: + workspace = WorkspaceClient() + assignee_id = find_account_id_by_email(workspace, connection, assignee_email) if assignee_email else None + optional_fields = { + "description": plain_text_to_jira_document(description) if description else None, + "assignee": {"accountId": assignee_id} if assignee_id else None, + "labels": labels, + "priority": {"name": priority} if priority else None, + } + issue_fields: dict[str, Any] = { + **parse_other_fields(fields), + "project": {"key": project_key}, + "summary": summary, + "issuetype": {"name": issue_type}, + **{key: value for key, value in optional_fields.items() if value}, + } + + result = send_jira_request( + workspace, connection, ExternalFunctionRequestHttpMethod.POST, "issue", {"fields": issue_fields} + ) + print(f"Created issue {result['key']}") + workspace.dbutils.jobs.taskValues.set(key="issue_key", value=result["key"]) + + +def find_account_id_by_email(workspace: WorkspaceClient, connection: str, email: str) -> str: + users = send_jira_request( + workspace, connection, ExternalFunctionRequestHttpMethod.GET, "user/search", None, params={"query": email} + ) + # Jira searches by prefix and returns a null email for users with a private profile. + matches = [user for user in users if (user.get("emailAddress") or "").lower() == email.lower()] + if len(matches) != 1: + raise ValueError(f"Found {len(matches)} Jira users with email {email}. Enter the email of exactly one user.") + return matches[0]["accountId"] + + +def parse_other_fields(fields: Optional[str]) -> dict[str, Any]: + try: + parsed = json.loads(fields) if fields else {} + except json.JSONDecodeError as error: + raise ValueError(f"Other fields must be a JSON object: {error}") from None + if not isinstance(parsed, dict): + raise ValueError("Other fields must be a JSON object, not a list or a single value") + return parsed + + +def send_jira_request( + workspace: WorkspaceClient, + connection: str, + method: ExternalFunctionRequestHttpMethod, + path: str, + body: Optional[dict[str, Any]], + params: Optional[dict[str, str]] = None, +) -> Any: + response = workspace.serving_endpoints.http_request( + conn=connection, method=method, path=path, json=body, params=params + ) + try: + return response.json() + except ValueError: + raise RuntimeError(f"Jira returned a non-JSON response: {response.text}") from None + + +def plain_text_to_jira_document(text: str) -> dict[str, Any]: + # Jira's v3 API only takes rich text as an Atlassian Document Format document. + paragraphs = [ + {"type": "paragraph", "content": [{"type": "text", "text": line}]} for line in text.splitlines() if line.strip() + ] + return {"type": "doc", "version": 1, "content": paragraphs} diff --git a/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/send_slack_message.py b/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/send_slack_message.py new file mode 100644 index 00000000000..734f277ace5 --- /dev/null +++ b/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/send_slack_message.py @@ -0,0 +1,91 @@ +import json +from typing import Annotated, Any, Optional + +from databricks.lakeflow.integrations import ( + HttpConnectionFilter, + HttpUcConnectionParameter, + integration, + ParameterMetadata, + TextAreaParameter, +) +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.serving import ExternalFunctionRequestHttpMethod + +SlackConnection = Annotated[HttpUcConnectionParameter, HttpConnectionFilter(host="https://slack.com", base_path="/api")] + + +@integration( + display_name="Send Slack message", + description="Posts a message to a Slack channel or direct message with optional rich formatting", + icon="send", + promoted_parameter="channel", + parameters=dict( + connection=ParameterMetadata( + display_name="Connection", + description="Unity Catalog connection to the Slack API. Its bot token needs the chat:write scope.", + ), + channel=ParameterMetadata(display_name="Channel", description="Channel name or ID. For example, #alerts."), + message=ParameterMetadata( + display_name="Message", + description="Text to post, in plain text or Slack markdown. Notifications show it when you also set blocks.", + ), + blocks=ParameterMetadata( + display_name="Blocks", description="Slack Block Kit layout, in JSON, for richer formatting." + ), + thread_ts=ParameterMetadata( + display_name="Thread timestamp", description="Timestamp of the message to reply to in its thread." + ), + username=ParameterMetadata( + display_name="Username", + description="Bot name shown on the message. Needs the chat:write.customize scope.", + ), + icon_url=ParameterMetadata( + display_name="Icon URL", + description="Image shown as the bot's avatar. Needs the chat:write.customize scope.", + ), + icon_emoji=ParameterMetadata( + display_name="Icon emoji", + description="Emoji shown as the bot's avatar. For example, :robot_face:. Needs the chat:write.customize scope.", + ), + ), +) +def send_slack_message( + connection: SlackConnection, + channel: str, + message: TextAreaParameter, + blocks: Optional[TextAreaParameter] = None, + thread_ts: Optional[str] = None, + username: Optional[str] = None, + icon_url: Optional[str] = None, + icon_emoji: Optional[str] = None, +) -> None: + optional_fields = { + "blocks": json.loads(blocks) if blocks else None, + "thread_ts": thread_ts, + "username": username, + "icon_emoji": icon_emoji, + "icon_url": None if icon_emoji else icon_url, + } + payload: dict[str, Any] = {"channel": channel, "text": message} + payload |= {key: value for key, value in optional_fields.items() if value} + + workspace = WorkspaceClient() + response = workspace.serving_endpoints.http_request( + conn=connection, + method=ExternalFunctionRequestHttpMethod.POST, + path="chat.postMessage", + json=payload, + ) + + try: + result = response.json() + except ValueError: + raise RuntimeError(f"Slack returned a non-JSON response: {response.text}") from None + + # Slack reports API errors with HTTP 200 and ok=false. + if not result.get("ok"): + raise RuntimeError(f"Slack API error: {result.get('error')}") + + print(f"Posted message {result['ts']} to channel {result['channel']}") + workspace.dbutils.jobs.taskValues.set(key="ts", value=result["ts"]) + workspace.dbutils.jobs.taskValues.set(key="channel_id", value=result["channel"]) diff --git a/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/slack.py b/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/slack.py deleted file mode 100644 index 8834a752e4f..00000000000 --- a/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/slack.py +++ /dev/null @@ -1,27 +0,0 @@ -import requests -from databricks.sdk import WorkspaceClient -from databricks.lakeflow.integrations import integration - - -@integration(display_name="Send Slack message") -def send_slack_message( - text: str, - conn_id: str, - channel: str, # Example: "C0A874NCL8N" -): - w = WorkspaceClient() - - response = requests.post( - f"{w.config.host}/api/2.0/unity-catalog/connections/{conn_id}/proxy/chat.postMessage", - headers={ - **w.config.authenticate(), - "Accept": "application/json", - "Content-Type": "application/json", - "Accept-Encoding": "identity", # ask for no compression - }, - json={ - "channel": channel, - "text": text, - }, - ) - print(response.json()) diff --git a/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/wait_for_rest_api.py b/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/wait_for_rest_api.py new file mode 100644 index 00000000000..608d415b3fc --- /dev/null +++ b/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/wait_for_rest_api.py @@ -0,0 +1,244 @@ +import json +import re +import time +from dataclasses import dataclass +from datetime import timedelta +from typing import Any, Optional +from urllib.parse import unquote, urlsplit + +from databricks.lakeflow.integrations import ( + Context, + HttpUcConnectionParameter, + integration, + ParameterMetadata, + PollResult, + Sensor, +) +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.serving import ExternalFunctionRequestHttpMethod as Method + +CONDITION = re.compile(r"(?P[\w.\[\]-]+)\s*(?P==|!=|\bin\b)\s*(?P.+)") +PREVIEW_LENGTH = 2000 + + +@dataclass(frozen=True) +class Condition: + text: str + response_path: str + operator: str + expected: Any + + +@dataclass(frozen=True) +class WaitSettings: + success_condition: Condition + failure_condition: Optional[Condition] + poll_interval_seconds: int + poll_timeout_seconds: int + output_mappings: dict[str, str] + + +@dataclass(frozen=True) +class Defer: + status: str + + +@dataclass(frozen=True) +class Complete: + outputs: dict[str, Any] + + +@integration( + display_name="Wait for REST API", + description="Polls a REST API until its response meets a condition, releasing compute between polls", + icon="clock", + promoted_parameter="path", + parameters=dict( + connection=ParameterMetadata(display_name="Connection", description="Unity Catalog connection to the API."), + path=ParameterMetadata( + display_name="Path", description="Status path after the connection's base path. For example, jobs/42." + ), + success_condition=ParameterMetadata( + display_name="Success condition", + description='Condition that ends the wait, using ==, !=, or in. For example, status == "done".', + ), + failure_condition=ParameterMetadata( + display_name="Failure condition", + description='Condition that fails the task. For example, status == "failed".', + ), + method=ParameterMetadata(display_name="Method", description="HTTP method of each poll."), + query_params=ParameterMetadata( + display_name="Query parameters", + description="Added to the status URL. For example, run_id with 42 adds ?run_id=42.", + ), + headers=ParameterMetadata(display_name="Headers", description="Extra headers to send with each poll."), + poll_interval_seconds=ParameterMetadata( + display_name="Poll interval in seconds", description="Wait between polls." + ), + poll_timeout_seconds=ParameterMetadata( + display_name="Timeout in seconds", description="How long to keep polling before the task fails." + ), + output_mappings=ParameterMetadata( + display_name="Output mappings", + description="Each key is a task value name and each value is its path in the final response. For " + "example, rows with result.rows.", + ), + ), +) +class WaitForRestApi(Sensor): + def __init__( + self, + connection: HttpUcConnectionParameter, + path: str, + success_condition: str, + failure_condition: Optional[str] = None, + method: Method = Method.GET, + query_params: Optional[dict[str, str]] = None, + headers: Optional[dict[str, str]] = None, + poll_interval_seconds: int = 30, + poll_timeout_seconds: int = 3600, + output_mappings: Optional[dict[str, str]] = None, + ) -> None: + if poll_interval_seconds < 0 or poll_timeout_seconds < 0: + raise ValueError("poll_interval_seconds and poll_timeout_seconds cannot be negative") + if ".." in unquote(urlsplit(path).path).split("/"): + raise ValueError(f"path can't contain '..' segments, got: {path}") + self.settings = WaitSettings( + success_condition=parse_condition("success_condition", success_condition, 'status == "done"'), + failure_condition=( + parse_condition("failure_condition", failure_condition, 'status == "failed"') + if failure_condition + else None + ), + poll_interval_seconds=poll_interval_seconds, + poll_timeout_seconds=poll_timeout_seconds, + output_mappings=output_mappings or {}, + ) + self.request = dict(conn=connection, method=method, path=path, params=query_params, headers=headers) + + def poll(self, ctx: Context) -> PollResult: + waited_seconds = time.time() - get_or_save_start_time(ctx) + + workspace = WorkspaceClient() + response = workspace.serving_endpoints.http_request(**self.request) + decision = decide_poll( + self.settings, + response=parse_response(response), + response_text=response.text, + waited_seconds=waited_seconds, + ) + + match decision: + case Defer(status): + print(status) + print(f"Polling again in {self.settings.poll_interval_seconds}s.") + return PollResult.deferred(timedelta(seconds=self.settings.poll_interval_seconds)) + case Complete(outputs): + print(f"{self.settings.success_condition.text} is met") + print(f"Response: {preview(response.text)}") + for name, value in outputs.items(): + workspace.dbutils.jobs.taskValues.set(key=name, value=value) + print(f"Saved task value {name}: {preview(json.dumps(value))}") + return PollResult.completed() + + +def decide_poll(settings: WaitSettings, response: Any, response_text: str, waited_seconds: float) -> Defer | Complete: + failure = settings.failure_condition + if failure and satisfies_condition(response, failure): + raise RuntimeError(f"failure_condition met: {failure.text}. Response: {preview(response_text)}") + + success = settings.success_condition + if satisfies_condition(response, success): + return Complete(extract_outputs(response, settings.output_mappings)) + + current_value = json.dumps(extract_nested_value(response, success.response_path)) + status = f"{success.text} is not met, {success.response_path} is {current_value}" + if waited_seconds + settings.poll_interval_seconds > settings.poll_timeout_seconds: + raise RuntimeError( + f"Timed out after {settings.poll_timeout_seconds}s. {status}. Response: {preview(response_text)}" + ) + + return Defer(status) + + +def get_or_save_start_time(ctx: Context) -> float: + # Deferring releases the compute, so the time the wait started is kept in a task value. + start_time = ctx.get_task_value("start_time", default=0) + if not start_time: + start_time = time.time() + ctx.set_task_value("start_time", start_time) + + return start_time + + +def parse_condition(name: str, text: str, example: str) -> Condition: + error = ValueError(f"{name} must look like {example}, got: {text}") + match = CONDITION.fullmatch(text.strip()) + if not match: + raise error + + try: + expected = json.loads(match["expected"]) + except json.JSONDecodeError: + raise error from None + + if match["operator"] == "in" and not isinstance(expected, list): + raise error + + return Condition(text, match["path"], match["operator"], expected) + + +def satisfies_condition(response: Any, condition: Condition) -> bool: + actual = extract_nested_value(response, condition.response_path) + # A missing field only meets a condition on null, so the wait goes on until the field appears. + if actual is None and condition.expected is not None: + return False + + candidates = condition.expected if condition.operator == "in" else [condition.expected] + found = any(json_equal(actual, candidate) for candidate in candidates) + return not found if condition.operator == "!=" else found + + +def json_equal(left: Any, right: Any) -> bool: + # Python treats True as 1, but JSON true and 1 are different values. + return left == right and isinstance(left, bool) == isinstance(right, bool) + + +def parse_response(response: Any) -> Any: + try: + return response.json() + except ValueError: + return response.text + + +def preview(text: str) -> str: + if not text: + return "(empty)" + if len(text) > PREVIEW_LENGTH: + return f"{text[:PREVIEW_LENGTH]}... (truncated)" + return text + + +def extract_outputs(response: Any, mappings: dict[str, str]) -> dict[str, Any]: + outputs = {} + for name, response_path in mappings.items(): + value = extract_nested_value(response, response_path) + if value is None: + raise RuntimeError(f"The response has no value at {response_path}") + + outputs[name] = value + return outputs + + +def extract_nested_value(response: Any, response_path: str) -> Any: + # result.items[0].id becomes the keys result, items, 0, and id. A leading [0] indexes a list response. + normalized_path = response_path.replace("[", ".").replace("]", "").lstrip(".") + value = response + for key in normalized_path.split("."): + if isinstance(value, dict): + value = value.get(key) + elif isinstance(value, list) and key.isdigit() and int(key) < len(value): + value = value[int(key)] + else: + return None + return value diff --git a/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/wait_for_run_sensor.py b/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/wait_for_run_sensor.py deleted file mode 100644 index 35e2c945d4a..00000000000 --- a/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/wait_for_run_sensor.py +++ /dev/null @@ -1,28 +0,0 @@ -import datetime - -from databricks.lakeflow.integrations import integration, Context, Sensor, SensorResult -from databricks.sdk import WorkspaceClient - - -@integration(display_name="Wait for run") -class WaitForRunSensor(Sensor): - run_id: str - - def __init__(self, run_id: str): - self.run_id = run_id - - def poll(self, ctx: Context) -> SensorResult: - from databricks.sdk.runtime import dbutils - - w = WorkspaceClient() - - run = w.jobs.get_run(run_id=int(self.run_id)) - result_state = run.state.result_state if run.state else None - - if result_state is not None: - dbutils.jobs.taskValues.set("result_state", result_state.name) - return SensorResult.completed() - else: - print("Running", run.run_page_url) - print("Waiting for a minute...") - return SensorResult.deferred(duration=datetime.timedelta(minutes=1)) diff --git a/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/wait_for_slack_reaction.py b/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/wait_for_slack_reaction.py new file mode 100644 index 00000000000..17306cc2340 --- /dev/null +++ b/acceptance/bundle/templates/lakeflow-integrations/output/my_lakeflow_integrations/src/my_lakeflow_integrations/examples/wait_for_slack_reaction.py @@ -0,0 +1,127 @@ +import datetime +from typing import Annotated, Any, Optional + +from databricks.lakeflow.integrations import ( + Context, + HttpConnectionFilter, + HttpUcConnectionParameter, + integration, + ParameterMetadata, + PollResult, + Sensor, +) +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.serving import ExternalFunctionRequestHttpMethod + +SlackConnection = Annotated[HttpUcConnectionParameter, HttpConnectionFilter(host="https://slack.com", base_path="/api")] + +MAX_RETRIES = 5 + + +@integration( + display_name="Wait for Slack reaction", + description="Pause until someone reacts to a Slack message", + icon="clock", + promoted_parameter="message_ts", + parameters=dict( + connection=ParameterMetadata( + display_name="Connection", + description="Unity Catalog connection to the Slack API. Its bot token needs the reactions:read scope.", + ), + channel_id=ParameterMetadata( + display_name="Channel ID", + description="For example, {{tasks.send_slack_message.values.channel_id}} from a Send Slack message task.", + ), + message_ts=ParameterMetadata( + display_name="Message timestamp", + description="Slack's ID for the message. For example, {{tasks.send_slack_message.values.ts}} from a Send Slack message task.", + ), + emojis=ParameterMetadata( + display_name="Emojis", + description="Slack emoji names. For example, white_check_mark. Leave empty for any reaction.", + ), + poll_interval_seconds=ParameterMetadata( + display_name="Poll interval (seconds)", description="How often to check for reactions." + ), + ), +) +class WaitForSlackReaction(Sensor): + def __init__( + self, + connection: SlackConnection, + channel_id: str, + message_ts: str, + emojis: Optional[list[str]] = None, + poll_interval_seconds: int = 60, + ): + if poll_interval_seconds < 0: + raise ValueError(f"Poll interval must be 0 seconds or more, got {poll_interval_seconds}.") + + self.connection = connection + self.channel_id = channel_id + self.message_ts = message_ts + self.emojis = {emoji_base_name(emoji.strip(":")) for emoji in emojis or [] if emoji.strip(":")} + self.poll_interval_seconds = poll_interval_seconds + + def poll(self, ctx: Context) -> PollResult: + workspace = WorkspaceClient() + try: + reactions = self.get_reactions(workspace) + except Exception as error: + return self.retry_or_fail(ctx, error) + + ctx.set_task_value("failed_polls", 0) + + reaction = find_matching_reaction(reactions, self.emojis) + if reaction is None: + print(f"No matching reaction yet, checking again in {self.poll_interval_seconds} seconds") + return PollResult.deferred(datetime.timedelta(seconds=self.poll_interval_seconds)) + + emoji, user_id = emoji_base_name(reaction["name"]), reaction["users"][0] + print(f"Found reaction :{emoji}: by user {user_id}") + workspace.dbutils.jobs.taskValues.set(key="reaction", value=emoji) + workspace.dbutils.jobs.taskValues.set(key="user_id", value=user_id) + return PollResult.completed() + + def get_reactions(self, workspace: WorkspaceClient) -> list[dict[str, Any]]: + response = workspace.serving_endpoints.http_request( + conn=self.connection, + method=ExternalFunctionRequestHttpMethod.GET, + path="reactions.get", + params={"channel": self.channel_id, "timestamp": self.message_ts, "full": "true"}, + ) + + try: + result = response.json() + except ValueError: + raise RuntimeError(f"Slack returned a non-JSON response: {response.text}") from None + + # Slack reports API errors with HTTP 200 and ok=false. + if not result.get("ok"): + raise RuntimeError( + f"Slack API error for message {self.message_ts} in channel {self.channel_id}: {result.get('error')}" + ) + return result["message"].get("reactions", []) + + def retry_or_fail(self, ctx: Context, error: Exception) -> PollResult: + failed_polls = ctx.get_task_value("failed_polls", 0) + 1 + if failed_polls > MAX_RETRIES: + raise RuntimeError(f"Checking for reactions failed {failed_polls} times in a row: {error}") from error + + ctx.set_task_value("failed_polls", failed_polls) + print( + f"Checking for reactions failed, retry {failed_polls} of {MAX_RETRIES} in {self.poll_interval_seconds} seconds: {error}" + ) + return PollResult.deferred(datetime.timedelta(seconds=self.poll_interval_seconds)) + + +def emoji_base_name(emoji: str) -> str: + return emoji.split("::")[0] + + +def find_matching_reaction(reactions: list[dict[str, Any]], emojis: set[str]) -> Optional[dict[str, Any]]: + # Slack lists reactions in the order they were first added, so the earliest match wins. + for reaction in reactions: + if not emojis or emoji_base_name(reaction["name"]) in emojis: + return reaction + return None diff --git a/acceptance/bundle/templates/lakeflow-integrations/script b/acceptance/bundle/templates/lakeflow-integrations/script index 0b0925e7cfc..79e8feabfea 100644 --- a/acceptance/bundle/templates/lakeflow-integrations/script +++ b/acceptance/bundle/templates/lakeflow-integrations/script @@ -8,16 +8,15 @@ cleanup() { rm -rf "$WHEELS_DIR" } -# The example modules import `requests` and `databricks.sdk` at module top level, -# but the acceptance harness runs uv offline against a fixed find-links dir that -# only has databricks-lakeflow-integrations. Build stub wheels for these two -# packages from `stub-wheels/` into a per-test find-links dir that also -# symlinks the real vendored wheels, and point UV_FIND_LINKS at it. +# The example modules import `databricks.sdk` at module top level, but the +# acceptance harness runs uv offline against a fixed find-links dir that only +# has databricks-lakeflow-integrations. Build a stub wheel for it from +# `stub-wheels/` into a per-test find-links dir that also symlinks the real +# vendored wheels, and point UV_FIND_LINKS at it. mkdir -p "$WHEELS_DIR" for whl in "$VENDORED_PY_PACKAGES"/*.whl; do ln -sf "$whl" "$WHEELS_DIR/$(basename "$whl")" done -uv build --wheel --quiet --out-dir "$WHEELS_DIR" "$STUB_WHEELS_SRC/requests" uv build --wheel --quiet --out-dir "$WHEELS_DIR" "$STUB_WHEELS_SRC/databricks-sdk" export UV_FIND_LINKS="$WHEELS_DIR" diff --git a/acceptance/bundle/templates/lakeflow-integrations/stub-wheels/databricks-sdk/src/databricks/sdk/runtime/__init__.py b/acceptance/bundle/templates/lakeflow-integrations/stub-wheels/databricks-sdk/src/databricks/sdk/runtime/__init__.py deleted file mode 100644 index beb784ac061..00000000000 --- a/acceptance/bundle/templates/lakeflow-integrations/stub-wheels/databricks-sdk/src/databricks/sdk/runtime/__init__.py +++ /dev/null @@ -1 +0,0 @@ -dbutils = None diff --git a/acceptance/bundle/templates/lakeflow-integrations/stub-wheels/requests/src/requests/__init__.py b/acceptance/bundle/templates/lakeflow-integrations/stub-wheels/databricks-sdk/src/databricks/sdk/service/__init__.py similarity index 100% rename from acceptance/bundle/templates/lakeflow-integrations/stub-wheels/requests/src/requests/__init__.py rename to acceptance/bundle/templates/lakeflow-integrations/stub-wheels/databricks-sdk/src/databricks/sdk/service/__init__.py diff --git a/acceptance/bundle/templates/lakeflow-integrations/stub-wheels/databricks-sdk/src/databricks/sdk/service/serving.py b/acceptance/bundle/templates/lakeflow-integrations/stub-wheels/databricks-sdk/src/databricks/sdk/service/serving.py new file mode 100644 index 00000000000..7990c6a7c27 --- /dev/null +++ b/acceptance/bundle/templates/lakeflow-integrations/stub-wheels/databricks-sdk/src/databricks/sdk/service/serving.py @@ -0,0 +1,9 @@ +from enum import Enum + + +class ExternalFunctionRequestHttpMethod(Enum): + DELETE = "DELETE" + GET = "GET" + PATCH = "PATCH" + POST = "POST" + PUT = "PUT" diff --git a/acceptance/bundle/templates/lakeflow-integrations/stub-wheels/requests/pyproject.toml b/acceptance/bundle/templates/lakeflow-integrations/stub-wheels/requests/pyproject.toml deleted file mode 100644 index db7519a09be..00000000000 --- a/acceptance/bundle/templates/lakeflow-integrations/stub-wheels/requests/pyproject.toml +++ /dev/null @@ -1,11 +0,0 @@ -[project] -name = "requests" -version = "0.0.0" -requires-python = ">=3.10" - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["src/requests"] diff --git a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/README.md.tmpl b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/README.md.tmpl index f1914db34d2..68da9075889 100644 --- a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/README.md.tmpl +++ b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/README.md.tmpl @@ -2,21 +2,29 @@ A Declarative Automation Bundles project that extends Lakeflow Jobs with custom integrations. -This bundle ships with runnable examples under `src/{{.project_name}}/examples/` and -corresponding job definitions under `resources/`: +This bundle ships with examples under `src/{{.project_name}}/examples/` that call external +services through Unity Catalog connections: | Example | Type | What it does | | --- | --- | --- | -| `slack.send_slack_message` | Function | Posts a message to Slack through a Unity Catalog connection. | -| `wait_for_run_sensor.WaitForRunSensor` | Sensor | Polls a job run until it reaches a terminal state. | +| `send_slack_message.send_slack_message` | Function | Posts a message to a Slack channel or direct message, with optional rich formatting. | +| `wait_for_slack_reaction.WaitForSlackReaction` | Sensor | Waits until someone reacts to a Slack message. | +| `create_jira_issue.create_jira_issue` | Function | Creates a Jira issue. | +| `call_rest_api.call_rest_api` | Function | Sends a request to a REST API and saves the response as task values. | +| `wait_for_rest_api.WaitForRestApi` | Sensor | Polls a REST API until its response meets a condition. | + +Define jobs that run them under `resources/`. ## Getting started 1. Open this bundle in a Databricks workspace. 2. Click the **deployments rocket** 🚀 in the left sidebar to open the **Deployments** panel, then click **Deploy**. -3. To run a deployed job, hover over the resource in the **Deployments** panel - and click **Run**. +3. To make the deployed integrations appear in the task palette, add them to `/Workspace/Users//.lakeflow_integrations.yml`. For example: + ```yaml + integrations: + - '/Workspace/Users//.bundle//dev/artifacts/.internal/*.yml' + ``` ## Documentation diff --git a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/pyproject.toml.tmpl b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/pyproject.toml.tmpl index df79b0849be..6ecb229655d 100644 --- a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/pyproject.toml.tmpl +++ b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/pyproject.toml.tmpl @@ -6,7 +6,7 @@ requires-python = ">=3.12,<3.13" dependencies = [ # Any dependencies for jobs and pipelines in this project can be added here # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies - "databricks-lakeflow-integrations>=0.0.3,<0.1", + "databricks-lakeflow-integrations>=0.0.6,<0.1", ] [tool.hatch.build.targets.wheel] @@ -17,7 +17,6 @@ packages = [ [dependency-groups] dev = [ # libraries part of serverless environment - "requests", "databricks-sdk" ] diff --git a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/resources/send_slack_message.job.yml.tmpl b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/resources/send_slack_message.job.yml.tmpl deleted file mode 100644 index e34209ec0c1..00000000000 --- a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/resources/send_slack_message.job.yml.tmpl +++ /dev/null @@ -1,24 +0,0 @@ -resources: - jobs: - send_slack_message: - name: "[example] send_slack_message" - - tasks: - - task_key: slack - environment_key: {{.project_name}}_environment - python_operator_task: - main: {{.project_name}}.examples.slack.send_slack_message - parameters: - - name: "conn_id" - value: "CHANGEME" - - name: "text" - value: "Hello, World" - - name: "channel" - value: "CHANGEME" - - environments: - - environment_key: {{.project_name}}_environment - spec: - environment_version: "5" - dependencies: - - ../dist/*.whl diff --git a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/resources/wait_for_run_sensor.job.yml.tmpl b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/resources/wait_for_run_sensor.job.yml.tmpl deleted file mode 100644 index 148c20d2135..00000000000 --- a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/resources/wait_for_run_sensor.job.yml.tmpl +++ /dev/null @@ -1,20 +0,0 @@ -resources: - jobs: - wait_for_run_sensor: - name: "[example] wait_for_run_sensor" - - tasks: - - task_key: wait_for_run_sensor - environment_key: {{.project_name}}_environment - python_operator_task: - main: {{.project_name}}.examples.wait_for_run_sensor.WaitForRunSensor - parameters: - - name: "run_id" - value: "CHANGEME" - - environments: - - environment_key: {{.project_name}}_environment - spec: - environment_version: "5" - dependencies: - - ../dist/*.whl diff --git a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/call_rest_api.py b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/call_rest_api.py new file mode 100644 index 00000000000..3f678a8662c --- /dev/null +++ b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/call_rest_api.py @@ -0,0 +1,111 @@ +import json +from typing import Any, Optional +from urllib.parse import unquote, urlsplit + +from databricks.lakeflow.integrations import ( + HttpUcConnectionParameter, + integration, + ParameterMetadata, + TextAreaParameter, +) +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.serving import ExternalFunctionRequestHttpMethod as Method + +PREVIEW_LENGTH = 2000 + + +@integration( + display_name="Call REST API", + description="Sends a request through a UC connection, saving the response as task values", + icon="globe", + promoted_parameter="path", + parameters=dict( + connection=ParameterMetadata(display_name="Connection", description="Unity Catalog connection to the API."), + path=ParameterMetadata( + display_name="Path", description="Path after the connection's base path. For example, jobs/42/refresh." + ), + method=ParameterMetadata(display_name="Method", description="HTTP method of the request."), + query_params=ParameterMetadata( + display_name="Query parameters", + description="Added to the request URL. For example, limit with 10 adds ?limit=10.", + ), + headers=ParameterMetadata(display_name="Headers", description="Extra headers to send with the request."), + body=ParameterMetadata(display_name="Body", description="Request body, in JSON."), + output_mappings=ParameterMetadata( + display_name="Output mappings", + description="Each key is a task value name and each value is its path in the response. For example, " + "job_id with id, or first_id with items[0].id.", + ), + ), +) +def call_rest_api( + connection: HttpUcConnectionParameter, + path: str, + method: Method = Method.GET, + query_params: Optional[dict[str, str]] = None, + headers: Optional[dict[str, str]] = None, + body: Optional[TextAreaParameter] = None, + output_mappings: Optional[dict[str, str]] = None, +) -> None: + if ".." in unquote(urlsplit(path).path).split("/"): + raise ValueError(f"path can't contain '..' segments, got: {path}") + json_body = parse_json_body(body) + + workspace = WorkspaceClient() + print(f"Sending {method.value} {path}") + response = workspace.serving_endpoints.http_request( + conn=connection, method=method, path=path, params=query_params, headers=headers, json=json_body + ) + print(f"Response: {preview(response.text)}") + + outputs = extract_outputs(parse_response(response), output_mappings or {}) + for name, value in outputs.items(): + workspace.dbutils.jobs.taskValues.set(key=name, value=value) + print(f"Saved task value {name}: {preview(json.dumps(value))}") + + +def parse_json_body(body: Optional[str]) -> Any: + try: + return json.loads(body) if body else None + except json.JSONDecodeError as error: + raise ValueError(f"body must be valid JSON: {error}") from None + + +def parse_response(response: Any) -> Any: + try: + return response.json() + except ValueError: + return response.text + + +def preview(text: str) -> str: + if not text: + return "(empty)" + if len(text) > PREVIEW_LENGTH: + return f"{text[:PREVIEW_LENGTH]}... (truncated)" + return text + + +def extract_outputs(response: Any, mappings: dict[str, str]) -> dict[str, Any]: + outputs = {} + for name, response_path in mappings.items(): + value = extract_nested_value(response, response_path) + if value is None: + raise RuntimeError(f"The response has no value at {response_path}") + + outputs[name] = value + return outputs + + +def extract_nested_value(response: Any, response_path: str) -> Any: + # result.items[0].id becomes the keys result, items, 0, and id. A leading [0] indexes a list response. + normalized_path = response_path.replace("[", ".").replace("]", "").lstrip(".") + value = response + for key in normalized_path.split("."): + if isinstance(value, dict): + value = value.get(key) + elif isinstance(value, list) and key.isdigit() and int(key) < len(value): + value = value[int(key)] + else: + return None + return value diff --git a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/create_jira_issue.py b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/create_jira_issue.py new file mode 100644 index 00000000000..91db7bf7fd0 --- /dev/null +++ b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/create_jira_issue.py @@ -0,0 +1,117 @@ +import json +from typing import Annotated, Any, Optional + +from databricks.lakeflow.integrations import ( + HttpConnectionFilter, + HttpUcConnectionParameter, + integration, + ParameterMetadata, + TextAreaParameter, +) +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.serving import ExternalFunctionRequestHttpMethod + +JiraConnection = Annotated[HttpUcConnectionParameter, HttpConnectionFilter(host="https://api.atlassian.com")] + + +@integration( + display_name="Create Jira issue", + description="Creates a Jira issue", + icon="plus-circle", + promoted_parameter="project_key", + parameters=dict( + connection=ParameterMetadata( + display_name="Connection", description="Unity Catalog connection to the Jira Cloud REST API." + ), + project_key=ParameterMetadata(display_name="Project key", description="For example, DATA."), + summary=ParameterMetadata(display_name="Summary", description="Title of the issue."), + issue_type=ParameterMetadata(display_name="Issue type", description="For example, Bug."), + description=ParameterMetadata(display_name="Description", description="Body of the issue, in plain text."), + assignee_email=ParameterMetadata(display_name="Assignee email", description="For example, ana@example.com."), + labels=ParameterMetadata(display_name="Labels", description="For example, pipeline."), + priority=ParameterMetadata(display_name="Priority", description="For example, High."), + fields=ParameterMetadata( + display_name="Other fields", + description="More Jira fields to set, as a JSON object of field IDs and values. " + 'For example, {"customfield_10000": "data-eng"}. The fields above take precedence.', + ), + ), +) +def create_jira_issue( + connection: JiraConnection, + project_key: str, + summary: str, + issue_type: str = "Task", + description: Optional[TextAreaParameter] = None, + assignee_email: Optional[str] = None, + labels: Optional[list[str]] = None, + priority: Optional[str] = None, + fields: Optional[TextAreaParameter] = None, +) -> None: + workspace = WorkspaceClient() + assignee_id = find_account_id_by_email(workspace, connection, assignee_email) if assignee_email else None + optional_fields = { + "description": plain_text_to_jira_document(description) if description else None, + "assignee": {"accountId": assignee_id} if assignee_id else None, + "labels": labels, + "priority": {"name": priority} if priority else None, + } + issue_fields: dict[str, Any] = { + **parse_other_fields(fields), + "project": {"key": project_key}, + "summary": summary, + "issuetype": {"name": issue_type}, + **{key: value for key, value in optional_fields.items() if value}, + } + + result = send_jira_request( + workspace, connection, ExternalFunctionRequestHttpMethod.POST, "issue", {"fields": issue_fields} + ) + print(f"Created issue {result['key']}") + workspace.dbutils.jobs.taskValues.set(key="issue_key", value=result["key"]) + + +def find_account_id_by_email(workspace: WorkspaceClient, connection: str, email: str) -> str: + users = send_jira_request( + workspace, connection, ExternalFunctionRequestHttpMethod.GET, "user/search", None, params={"query": email} + ) + # Jira searches by prefix and returns a null email for users with a private profile. + matches = [user for user in users if (user.get("emailAddress") or "").lower() == email.lower()] + if len(matches) != 1: + raise ValueError(f"Found {len(matches)} Jira users with email {email}. Enter the email of exactly one user.") + return matches[0]["accountId"] + + +def parse_other_fields(fields: Optional[str]) -> dict[str, Any]: + try: + parsed = json.loads(fields) if fields else {} + except json.JSONDecodeError as error: + raise ValueError(f"Other fields must be a JSON object: {error}") from None + if not isinstance(parsed, dict): + raise ValueError("Other fields must be a JSON object, not a list or a single value") + return parsed + + +def send_jira_request( + workspace: WorkspaceClient, + connection: str, + method: ExternalFunctionRequestHttpMethod, + path: str, + body: Optional[dict[str, Any]], + params: Optional[dict[str, str]] = None, +) -> Any: + response = workspace.serving_endpoints.http_request( + conn=connection, method=method, path=path, json=body, params=params + ) + try: + return response.json() + except ValueError: + raise RuntimeError(f"Jira returned a non-JSON response: {response.text}") from None + + +def plain_text_to_jira_document(text: str) -> dict[str, Any]: + # Jira's v3 API only takes rich text as an Atlassian Document Format document. + paragraphs = [ + {"type": "paragraph", "content": [{"type": "text", "text": line}]} for line in text.splitlines() if line.strip() + ] + return {"type": "doc", "version": 1, "content": paragraphs} diff --git a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/send_slack_message.py b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/send_slack_message.py new file mode 100644 index 00000000000..734f277ace5 --- /dev/null +++ b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/send_slack_message.py @@ -0,0 +1,91 @@ +import json +from typing import Annotated, Any, Optional + +from databricks.lakeflow.integrations import ( + HttpConnectionFilter, + HttpUcConnectionParameter, + integration, + ParameterMetadata, + TextAreaParameter, +) +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.serving import ExternalFunctionRequestHttpMethod + +SlackConnection = Annotated[HttpUcConnectionParameter, HttpConnectionFilter(host="https://slack.com", base_path="/api")] + + +@integration( + display_name="Send Slack message", + description="Posts a message to a Slack channel or direct message with optional rich formatting", + icon="send", + promoted_parameter="channel", + parameters=dict( + connection=ParameterMetadata( + display_name="Connection", + description="Unity Catalog connection to the Slack API. Its bot token needs the chat:write scope.", + ), + channel=ParameterMetadata(display_name="Channel", description="Channel name or ID. For example, #alerts."), + message=ParameterMetadata( + display_name="Message", + description="Text to post, in plain text or Slack markdown. Notifications show it when you also set blocks.", + ), + blocks=ParameterMetadata( + display_name="Blocks", description="Slack Block Kit layout, in JSON, for richer formatting." + ), + thread_ts=ParameterMetadata( + display_name="Thread timestamp", description="Timestamp of the message to reply to in its thread." + ), + username=ParameterMetadata( + display_name="Username", + description="Bot name shown on the message. Needs the chat:write.customize scope.", + ), + icon_url=ParameterMetadata( + display_name="Icon URL", + description="Image shown as the bot's avatar. Needs the chat:write.customize scope.", + ), + icon_emoji=ParameterMetadata( + display_name="Icon emoji", + description="Emoji shown as the bot's avatar. For example, :robot_face:. Needs the chat:write.customize scope.", + ), + ), +) +def send_slack_message( + connection: SlackConnection, + channel: str, + message: TextAreaParameter, + blocks: Optional[TextAreaParameter] = None, + thread_ts: Optional[str] = None, + username: Optional[str] = None, + icon_url: Optional[str] = None, + icon_emoji: Optional[str] = None, +) -> None: + optional_fields = { + "blocks": json.loads(blocks) if blocks else None, + "thread_ts": thread_ts, + "username": username, + "icon_emoji": icon_emoji, + "icon_url": None if icon_emoji else icon_url, + } + payload: dict[str, Any] = {"channel": channel, "text": message} + payload |= {key: value for key, value in optional_fields.items() if value} + + workspace = WorkspaceClient() + response = workspace.serving_endpoints.http_request( + conn=connection, + method=ExternalFunctionRequestHttpMethod.POST, + path="chat.postMessage", + json=payload, + ) + + try: + result = response.json() + except ValueError: + raise RuntimeError(f"Slack returned a non-JSON response: {response.text}") from None + + # Slack reports API errors with HTTP 200 and ok=false. + if not result.get("ok"): + raise RuntimeError(f"Slack API error: {result.get('error')}") + + print(f"Posted message {result['ts']} to channel {result['channel']}") + workspace.dbutils.jobs.taskValues.set(key="ts", value=result["ts"]) + workspace.dbutils.jobs.taskValues.set(key="channel_id", value=result["channel"]) diff --git a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/slack.py b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/slack.py deleted file mode 100644 index 8834a752e4f..00000000000 --- a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/slack.py +++ /dev/null @@ -1,27 +0,0 @@ -import requests -from databricks.sdk import WorkspaceClient -from databricks.lakeflow.integrations import integration - - -@integration(display_name="Send Slack message") -def send_slack_message( - text: str, - conn_id: str, - channel: str, # Example: "C0A874NCL8N" -): - w = WorkspaceClient() - - response = requests.post( - f"{w.config.host}/api/2.0/unity-catalog/connections/{conn_id}/proxy/chat.postMessage", - headers={ - **w.config.authenticate(), - "Accept": "application/json", - "Content-Type": "application/json", - "Accept-Encoding": "identity", # ask for no compression - }, - json={ - "channel": channel, - "text": text, - }, - ) - print(response.json()) diff --git a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/wait_for_rest_api.py b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/wait_for_rest_api.py new file mode 100644 index 00000000000..608d415b3fc --- /dev/null +++ b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/wait_for_rest_api.py @@ -0,0 +1,244 @@ +import json +import re +import time +from dataclasses import dataclass +from datetime import timedelta +from typing import Any, Optional +from urllib.parse import unquote, urlsplit + +from databricks.lakeflow.integrations import ( + Context, + HttpUcConnectionParameter, + integration, + ParameterMetadata, + PollResult, + Sensor, +) +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.serving import ExternalFunctionRequestHttpMethod as Method + +CONDITION = re.compile(r"(?P[\w.\[\]-]+)\s*(?P==|!=|\bin\b)\s*(?P.+)") +PREVIEW_LENGTH = 2000 + + +@dataclass(frozen=True) +class Condition: + text: str + response_path: str + operator: str + expected: Any + + +@dataclass(frozen=True) +class WaitSettings: + success_condition: Condition + failure_condition: Optional[Condition] + poll_interval_seconds: int + poll_timeout_seconds: int + output_mappings: dict[str, str] + + +@dataclass(frozen=True) +class Defer: + status: str + + +@dataclass(frozen=True) +class Complete: + outputs: dict[str, Any] + + +@integration( + display_name="Wait for REST API", + description="Polls a REST API until its response meets a condition, releasing compute between polls", + icon="clock", + promoted_parameter="path", + parameters=dict( + connection=ParameterMetadata(display_name="Connection", description="Unity Catalog connection to the API."), + path=ParameterMetadata( + display_name="Path", description="Status path after the connection's base path. For example, jobs/42." + ), + success_condition=ParameterMetadata( + display_name="Success condition", + description='Condition that ends the wait, using ==, !=, or in. For example, status == "done".', + ), + failure_condition=ParameterMetadata( + display_name="Failure condition", + description='Condition that fails the task. For example, status == "failed".', + ), + method=ParameterMetadata(display_name="Method", description="HTTP method of each poll."), + query_params=ParameterMetadata( + display_name="Query parameters", + description="Added to the status URL. For example, run_id with 42 adds ?run_id=42.", + ), + headers=ParameterMetadata(display_name="Headers", description="Extra headers to send with each poll."), + poll_interval_seconds=ParameterMetadata( + display_name="Poll interval in seconds", description="Wait between polls." + ), + poll_timeout_seconds=ParameterMetadata( + display_name="Timeout in seconds", description="How long to keep polling before the task fails." + ), + output_mappings=ParameterMetadata( + display_name="Output mappings", + description="Each key is a task value name and each value is its path in the final response. For " + "example, rows with result.rows.", + ), + ), +) +class WaitForRestApi(Sensor): + def __init__( + self, + connection: HttpUcConnectionParameter, + path: str, + success_condition: str, + failure_condition: Optional[str] = None, + method: Method = Method.GET, + query_params: Optional[dict[str, str]] = None, + headers: Optional[dict[str, str]] = None, + poll_interval_seconds: int = 30, + poll_timeout_seconds: int = 3600, + output_mappings: Optional[dict[str, str]] = None, + ) -> None: + if poll_interval_seconds < 0 or poll_timeout_seconds < 0: + raise ValueError("poll_interval_seconds and poll_timeout_seconds cannot be negative") + if ".." in unquote(urlsplit(path).path).split("/"): + raise ValueError(f"path can't contain '..' segments, got: {path}") + self.settings = WaitSettings( + success_condition=parse_condition("success_condition", success_condition, 'status == "done"'), + failure_condition=( + parse_condition("failure_condition", failure_condition, 'status == "failed"') + if failure_condition + else None + ), + poll_interval_seconds=poll_interval_seconds, + poll_timeout_seconds=poll_timeout_seconds, + output_mappings=output_mappings or {}, + ) + self.request = dict(conn=connection, method=method, path=path, params=query_params, headers=headers) + + def poll(self, ctx: Context) -> PollResult: + waited_seconds = time.time() - get_or_save_start_time(ctx) + + workspace = WorkspaceClient() + response = workspace.serving_endpoints.http_request(**self.request) + decision = decide_poll( + self.settings, + response=parse_response(response), + response_text=response.text, + waited_seconds=waited_seconds, + ) + + match decision: + case Defer(status): + print(status) + print(f"Polling again in {self.settings.poll_interval_seconds}s.") + return PollResult.deferred(timedelta(seconds=self.settings.poll_interval_seconds)) + case Complete(outputs): + print(f"{self.settings.success_condition.text} is met") + print(f"Response: {preview(response.text)}") + for name, value in outputs.items(): + workspace.dbutils.jobs.taskValues.set(key=name, value=value) + print(f"Saved task value {name}: {preview(json.dumps(value))}") + return PollResult.completed() + + +def decide_poll(settings: WaitSettings, response: Any, response_text: str, waited_seconds: float) -> Defer | Complete: + failure = settings.failure_condition + if failure and satisfies_condition(response, failure): + raise RuntimeError(f"failure_condition met: {failure.text}. Response: {preview(response_text)}") + + success = settings.success_condition + if satisfies_condition(response, success): + return Complete(extract_outputs(response, settings.output_mappings)) + + current_value = json.dumps(extract_nested_value(response, success.response_path)) + status = f"{success.text} is not met, {success.response_path} is {current_value}" + if waited_seconds + settings.poll_interval_seconds > settings.poll_timeout_seconds: + raise RuntimeError( + f"Timed out after {settings.poll_timeout_seconds}s. {status}. Response: {preview(response_text)}" + ) + + return Defer(status) + + +def get_or_save_start_time(ctx: Context) -> float: + # Deferring releases the compute, so the time the wait started is kept in a task value. + start_time = ctx.get_task_value("start_time", default=0) + if not start_time: + start_time = time.time() + ctx.set_task_value("start_time", start_time) + + return start_time + + +def parse_condition(name: str, text: str, example: str) -> Condition: + error = ValueError(f"{name} must look like {example}, got: {text}") + match = CONDITION.fullmatch(text.strip()) + if not match: + raise error + + try: + expected = json.loads(match["expected"]) + except json.JSONDecodeError: + raise error from None + + if match["operator"] == "in" and not isinstance(expected, list): + raise error + + return Condition(text, match["path"], match["operator"], expected) + + +def satisfies_condition(response: Any, condition: Condition) -> bool: + actual = extract_nested_value(response, condition.response_path) + # A missing field only meets a condition on null, so the wait goes on until the field appears. + if actual is None and condition.expected is not None: + return False + + candidates = condition.expected if condition.operator == "in" else [condition.expected] + found = any(json_equal(actual, candidate) for candidate in candidates) + return not found if condition.operator == "!=" else found + + +def json_equal(left: Any, right: Any) -> bool: + # Python treats True as 1, but JSON true and 1 are different values. + return left == right and isinstance(left, bool) == isinstance(right, bool) + + +def parse_response(response: Any) -> Any: + try: + return response.json() + except ValueError: + return response.text + + +def preview(text: str) -> str: + if not text: + return "(empty)" + if len(text) > PREVIEW_LENGTH: + return f"{text[:PREVIEW_LENGTH]}... (truncated)" + return text + + +def extract_outputs(response: Any, mappings: dict[str, str]) -> dict[str, Any]: + outputs = {} + for name, response_path in mappings.items(): + value = extract_nested_value(response, response_path) + if value is None: + raise RuntimeError(f"The response has no value at {response_path}") + + outputs[name] = value + return outputs + + +def extract_nested_value(response: Any, response_path: str) -> Any: + # result.items[0].id becomes the keys result, items, 0, and id. A leading [0] indexes a list response. + normalized_path = response_path.replace("[", ".").replace("]", "").lstrip(".") + value = response + for key in normalized_path.split("."): + if isinstance(value, dict): + value = value.get(key) + elif isinstance(value, list) and key.isdigit() and int(key) < len(value): + value = value[int(key)] + else: + return None + return value diff --git a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/wait_for_run_sensor.py b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/wait_for_run_sensor.py deleted file mode 100644 index 35e2c945d4a..00000000000 --- a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/wait_for_run_sensor.py +++ /dev/null @@ -1,28 +0,0 @@ -import datetime - -from databricks.lakeflow.integrations import integration, Context, Sensor, SensorResult -from databricks.sdk import WorkspaceClient - - -@integration(display_name="Wait for run") -class WaitForRunSensor(Sensor): - run_id: str - - def __init__(self, run_id: str): - self.run_id = run_id - - def poll(self, ctx: Context) -> SensorResult: - from databricks.sdk.runtime import dbutils - - w = WorkspaceClient() - - run = w.jobs.get_run(run_id=int(self.run_id)) - result_state = run.state.result_state if run.state else None - - if result_state is not None: - dbutils.jobs.taskValues.set("result_state", result_state.name) - return SensorResult.completed() - else: - print("Running", run.run_page_url) - print("Waiting for a minute...") - return SensorResult.deferred(duration=datetime.timedelta(minutes=1)) diff --git a/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/wait_for_slack_reaction.py b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/wait_for_slack_reaction.py new file mode 100644 index 00000000000..17306cc2340 --- /dev/null +++ b/libs/template/templates/lakeflow-integrations/template/{{.project_name}}/src/{{.project_name}}/examples/wait_for_slack_reaction.py @@ -0,0 +1,127 @@ +import datetime +from typing import Annotated, Any, Optional + +from databricks.lakeflow.integrations import ( + Context, + HttpConnectionFilter, + HttpUcConnectionParameter, + integration, + ParameterMetadata, + PollResult, + Sensor, +) +from databricks.sdk import WorkspaceClient +from databricks.sdk.service.serving import ExternalFunctionRequestHttpMethod + +SlackConnection = Annotated[HttpUcConnectionParameter, HttpConnectionFilter(host="https://slack.com", base_path="/api")] + +MAX_RETRIES = 5 + + +@integration( + display_name="Wait for Slack reaction", + description="Pause until someone reacts to a Slack message", + icon="clock", + promoted_parameter="message_ts", + parameters=dict( + connection=ParameterMetadata( + display_name="Connection", + description="Unity Catalog connection to the Slack API. Its bot token needs the reactions:read scope.", + ), + channel_id=ParameterMetadata( + display_name="Channel ID", + description="For example, {{tasks.send_slack_message.values.channel_id}} from a Send Slack message task.", + ), + message_ts=ParameterMetadata( + display_name="Message timestamp", + description="Slack's ID for the message. For example, {{tasks.send_slack_message.values.ts}} from a Send Slack message task.", + ), + emojis=ParameterMetadata( + display_name="Emojis", + description="Slack emoji names. For example, white_check_mark. Leave empty for any reaction.", + ), + poll_interval_seconds=ParameterMetadata( + display_name="Poll interval (seconds)", description="How often to check for reactions." + ), + ), +) +class WaitForSlackReaction(Sensor): + def __init__( + self, + connection: SlackConnection, + channel_id: str, + message_ts: str, + emojis: Optional[list[str]] = None, + poll_interval_seconds: int = 60, + ): + if poll_interval_seconds < 0: + raise ValueError(f"Poll interval must be 0 seconds or more, got {poll_interval_seconds}.") + + self.connection = connection + self.channel_id = channel_id + self.message_ts = message_ts + self.emojis = {emoji_base_name(emoji.strip(":")) for emoji in emojis or [] if emoji.strip(":")} + self.poll_interval_seconds = poll_interval_seconds + + def poll(self, ctx: Context) -> PollResult: + workspace = WorkspaceClient() + try: + reactions = self.get_reactions(workspace) + except Exception as error: + return self.retry_or_fail(ctx, error) + + ctx.set_task_value("failed_polls", 0) + + reaction = find_matching_reaction(reactions, self.emojis) + if reaction is None: + print(f"No matching reaction yet, checking again in {self.poll_interval_seconds} seconds") + return PollResult.deferred(datetime.timedelta(seconds=self.poll_interval_seconds)) + + emoji, user_id = emoji_base_name(reaction["name"]), reaction["users"][0] + print(f"Found reaction :{emoji}: by user {user_id}") + workspace.dbutils.jobs.taskValues.set(key="reaction", value=emoji) + workspace.dbutils.jobs.taskValues.set(key="user_id", value=user_id) + return PollResult.completed() + + def get_reactions(self, workspace: WorkspaceClient) -> list[dict[str, Any]]: + response = workspace.serving_endpoints.http_request( + conn=self.connection, + method=ExternalFunctionRequestHttpMethod.GET, + path="reactions.get", + params={"channel": self.channel_id, "timestamp": self.message_ts, "full": "true"}, + ) + + try: + result = response.json() + except ValueError: + raise RuntimeError(f"Slack returned a non-JSON response: {response.text}") from None + + # Slack reports API errors with HTTP 200 and ok=false. + if not result.get("ok"): + raise RuntimeError( + f"Slack API error for message {self.message_ts} in channel {self.channel_id}: {result.get('error')}" + ) + return result["message"].get("reactions", []) + + def retry_or_fail(self, ctx: Context, error: Exception) -> PollResult: + failed_polls = ctx.get_task_value("failed_polls", 0) + 1 + if failed_polls > MAX_RETRIES: + raise RuntimeError(f"Checking for reactions failed {failed_polls} times in a row: {error}") from error + + ctx.set_task_value("failed_polls", failed_polls) + print( + f"Checking for reactions failed, retry {failed_polls} of {MAX_RETRIES} in {self.poll_interval_seconds} seconds: {error}" + ) + return PollResult.deferred(datetime.timedelta(seconds=self.poll_interval_seconds)) + + +def emoji_base_name(emoji: str) -> str: + return emoji.split("::")[0] + + +def find_matching_reaction(reactions: list[dict[str, Any]], emojis: set[str]) -> Optional[dict[str, Any]]: + # Slack lists reactions in the order they were first added, so the earliest match wins. + for reaction in reactions: + if not emojis or emoji_base_name(reaction["name"]) in emojis: + return reaction + return None diff --git a/libs/vendored_py_packages/databricks_lakeflow_integrations-0.0.3-py3-none-any.whl b/libs/vendored_py_packages/databricks_lakeflow_integrations-0.0.3-py3-none-any.whl deleted file mode 100644 index d3179c0096e..00000000000 Binary files a/libs/vendored_py_packages/databricks_lakeflow_integrations-0.0.3-py3-none-any.whl and /dev/null differ diff --git a/libs/vendored_py_packages/databricks_lakeflow_integrations-0.0.6-py3-none-any.whl b/libs/vendored_py_packages/databricks_lakeflow_integrations-0.0.6-py3-none-any.whl new file mode 100644 index 00000000000..f2b8159bff8 Binary files /dev/null and b/libs/vendored_py_packages/databricks_lakeflow_integrations-0.0.6-py3-none-any.whl differ