Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "uipath-langchain"
version = "0.20.6"
version = "0.20.7"
description = "Python SDK that enables developers to build and deploy LangGraph agents to the UiPath Cloud Platform"
readme = { file = "README.md", content-type = "text/markdown" }
requires-python = ">=3.11"
Expand Down
37 changes: 35 additions & 2 deletions src/uipath_langchain/guardrails/middlewares/_spans.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from uipath.platform.guardrails.decorators._exceptions import GuardrailBlockException

GUARDRAIL_ACTION = "guardrailAction"
GUARDRAIL_EVALUATION = "guardrailEvaluation"


def _new_span(name: str, span_type: str, **attributes: Any) -> Span:
Expand Down Expand Up @@ -52,6 +53,12 @@ def _error_text(exc: BaseException) -> str:
return str(exc)


def span_guid(span: Span) -> str:
"""``span``'s id as a GUID, the form ``guardrailSpanId`` uses."""
span_id = format(span.get_span_context().span_id, "016x")
return f"00000000-0000-0000-{span_id[:4]}-{span_id[4:]}"


def action_name(action: Any) -> str:
"""``BlockAction`` -> ``Block``."""
name = type(action).__name__
Expand All @@ -60,7 +67,10 @@ def action_name(action: Any) -> str:

@contextmanager
def guardrail_action_span(
guardrail_name: str, action: Any, result: GuardrailValidationResult
guardrail_name: str,
action: Any,
result: GuardrailValidationResult,
evaluation_span_id: str | None = None,
) -> Generator[Span]:
"""Record the action taken on a failed ``result``."""
severity = getattr(action, "severity_level", None)
Expand All @@ -69,7 +79,7 @@ def guardrail_action_span(
GUARDRAIL_ACTION,
guardrailName=guardrail_name,
action=action_name(action),
guardrailSpanId=result.span_id,
guardrailSpanId=evaluation_span_id or result.span_id,
reason=result.reason or None,
severityLevel=(
severity.name.title()
Expand All @@ -79,3 +89,26 @@ def guardrail_action_span(
)
with _active(span):
yield span


@contextmanager
def guardrail_evaluation_span(
guardrail_name: str, description: str | None = None
) -> Generator[Span]:
"""Record a rule check that runs in the agent."""
span = _new_span(
guardrail_name,
GUARDRAIL_EVALUATION,
guardrailName=guardrail_name,
guardrailDescription=description,
guardrailType="custom",
)
with _active(span):
yield span


def set_validation_result(span: Span, result: GuardrailValidationResult) -> None:
"""Record ``result``'s outcome on its evaluation span."""
span.set_attribute("validationResult", result.result.name)
if result.reason:
span.set_attribute("details", [result.reason])
30 changes: 18 additions & 12 deletions src/uipath_langchain/guardrails/middlewares/deterministic.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,12 @@

from ..enums import GuardrailExecutionStage
from ..models import GuardrailAction
from ._spans import guardrail_action_span
from ._spans import (
guardrail_action_span,
guardrail_evaluation_span,
set_validation_result,
span_guid,
)
from ._utils import (
convert_block_exception,
create_modified_tool_request,
Expand Down Expand Up @@ -235,18 +240,16 @@ def _evaluate_and_apply_modifications(
ToolCallRequest | None, dict[str, Any] | None, ToolMessage | Command[Any] | None
]:
"""Evaluate rules and apply modifications if validation fails."""
if not rules:
result = GuardrailValidationResult(
result=GuardrailValidationResultType.VALIDATION_FAILED,
reason="Empty rules - always apply action",
)
else:
with guardrail_evaluation_span(self._name, self._description) as span:
result = self._evaluate_rules(rules, stage, input_data, output_data)
if result.result != GuardrailValidationResultType.VALIDATION_FAILED:
return request, input_data, tool_result
set_validation_result(span, result)
if result.result != GuardrailValidationResultType.VALIDATION_FAILED:
return request, input_data, tool_result

try:
modified_data = self._handle_validation_result(result, data_for_action)
modified_data = self._handle_validation_result(
result, data_for_action, evaluation_span_id=span_guid(span)
)
except GuardrailBlockException as exc:
raise convert_block_exception(exc) from exc
except AgentRuntimeError:
Expand Down Expand Up @@ -383,10 +386,13 @@ def _extract_tool_output_data(
return {"output": content}

def _handle_validation_result(
self, result: GuardrailValidationResult, input_data: str | dict[str, Any]
self,
result: GuardrailValidationResult,
input_data: str | dict[str, Any],
evaluation_span_id: str | None = None,
) -> str | dict[str, Any] | None:
"""Handle guardrail validation result."""
if result.result != GuardrailValidationResultType.VALIDATION_FAILED:
return None
with guardrail_action_span(self._name, self.action, result):
with guardrail_action_span(self._name, self.action, result, evaluation_span_id):
return self.action.handle_validation_result(result, input_data, self._name)
101 changes: 101 additions & 0 deletions tests/guardrails/middlewares/test_deterministic_spans.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
"""Deterministic rules record their check as a guardrailEvaluation span."""

from collections.abc import Iterator
from typing import Any
from unittest.mock import MagicMock

import pytest
from langchain_core.messages import ToolMessage
from langgraph.prebuilt.tool_node import ToolCallRequest
from opentelemetry import trace
from opentelemetry.sdk.trace import ReadableSpan, TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
InMemorySpanExporter,
)
from opentelemetry.trace import StatusCode
from uipath.platform.guardrails.decorators import BlockAction, LogAction

from uipath_langchain.agent.exceptions import AgentRuntimeError
from uipath_langchain.guardrails import GuardrailAction, GuardrailExecutionStage
from uipath_langchain.guardrails.middlewares import (
UiPathDeterministicGuardrailMiddleware,
)


@pytest.fixture
def spans(monkeypatch: pytest.MonkeyPatch) -> Iterator[InMemorySpanExporter]:
provider = TracerProvider()
exporter = InMemorySpanExporter()
provider.add_span_processor(SimpleSpanProcessor(exporter))
monkeypatch.setattr(trace, "get_tracer", provider.get_tracer)
yield exporter
provider.shutdown()


async def _call_weather(action: GuardrailAction, city: str) -> None:
rule = UiPathDeterministicGuardrailMiddleware(
tools=["get_weather"],
rules=[lambda args: args.get("city") == "London"],
action=action,
stage=GuardrailExecutionStage.PRE,
name="London Rule",
description="Flags London",
)
[wrapper] = list(rule)
request = ToolCallRequest(
tool_call={"id": "tc1", "name": "get_weather", "args": {"city": city}},
tool=MagicMock(),
state={},
runtime=MagicMock(),
)

async def tool(request: ToolCallRequest) -> ToolMessage:
return ToolMessage(content="sunny", tool_call_id="tc1")

await wrapper.awrap_tool_call(request, tool)


def _named(spans: InMemorySpanExporter, name: str) -> ReadableSpan:
[span] = [s for s in spans.get_finished_spans() if s.name == name]
return span


def _attributes(span: ReadableSpan) -> dict[str, Any]:
return dict(span.attributes or {})


async def test_a_failed_rule_records_its_evaluation_and_links_the_action(
spans: InMemorySpanExporter,
) -> None:
await _call_weather(LogAction(), "London")

evaluation = _attributes(_named(spans, "London Rule"))
assert evaluation["span_type"] == "guardrailEvaluation"
assert evaluation["guardrailType"] == "custom"
assert evaluation["guardrailDescription"] == "Flags London"
assert evaluation["validationResult"] == "VALIDATION_FAILED"
assert evaluation["details"] == ("Rule 1 detected violation",)
span_id = format(_named(spans, "London Rule").context.span_id, "016x")
log = _attributes(_named(spans, "Log"))
assert log["guardrailSpanId"] == f"00000000-0000-0000-{span_id[:4]}-{span_id[4:]}"


async def test_a_passing_rule_records_its_evaluation_only(
spans: InMemorySpanExporter,
) -> None:
await _call_weather(LogAction(), "Paris")

evaluation = _attributes(_named(spans, "London Rule"))
assert evaluation["validationResult"] == "PASSED"
assert [s.name for s in spans.get_finished_spans()] == ["London Rule"]


async def test_a_blocking_rule_marks_its_action_as_failed(
spans: InMemorySpanExporter,
) -> None:
with pytest.raises(AgentRuntimeError):
await _call_weather(BlockAction(), "London")

assert _named(spans, "Block").status.status_code == StatusCode.ERROR
assert _named(spans, "London Rule").status.status_code == StatusCode.OK
2 changes: 1 addition & 1 deletion uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading