diff --git a/pyproject.toml b/pyproject.toml index da72725f1..e29eb473f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath-langchain" -version = "0.20.6" +version = "0.20.7" description = "Python SDK that enables developers to build and deploy LangGraph agents to the UiPath Cloud Platform" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/src/uipath_langchain/guardrails/middlewares/_spans.py b/src/uipath_langchain/guardrails/middlewares/_spans.py index c623c25d6..36c8eba19 100644 --- a/src/uipath_langchain/guardrails/middlewares/_spans.py +++ b/src/uipath_langchain/guardrails/middlewares/_spans.py @@ -13,6 +13,7 @@ from uipath.platform.guardrails.decorators._exceptions import GuardrailBlockException GUARDRAIL_ACTION = "guardrailAction" +GUARDRAIL_EVALUATION = "guardrailEvaluation" def _new_span(name: str, span_type: str, **attributes: Any) -> Span: @@ -52,6 +53,12 @@ def _error_text(exc: BaseException) -> str: return str(exc) +def span_guid(span: Span) -> str: + """``span``'s id as a GUID, the form ``guardrailSpanId`` uses.""" + span_id = format(span.get_span_context().span_id, "016x") + return f"00000000-0000-0000-{span_id[:4]}-{span_id[4:]}" + + def action_name(action: Any) -> str: """``BlockAction`` -> ``Block``.""" name = type(action).__name__ @@ -60,7 +67,10 @@ def action_name(action: Any) -> str: @contextmanager def guardrail_action_span( - guardrail_name: str, action: Any, result: GuardrailValidationResult + guardrail_name: str, + action: Any, + result: GuardrailValidationResult, + evaluation_span_id: str | None = None, ) -> Generator[Span]: """Record the action taken on a failed ``result``.""" severity = getattr(action, "severity_level", None) @@ -69,7 +79,7 @@ def guardrail_action_span( GUARDRAIL_ACTION, guardrailName=guardrail_name, action=action_name(action), - guardrailSpanId=result.span_id, + guardrailSpanId=evaluation_span_id or result.span_id, reason=result.reason or None, severityLevel=( severity.name.title() @@ -79,3 +89,26 @@ def guardrail_action_span( ) with _active(span): yield span + + +@contextmanager +def guardrail_evaluation_span( + guardrail_name: str, description: str | None = None +) -> Generator[Span]: + """Record a rule check that runs in the agent.""" + span = _new_span( + guardrail_name, + GUARDRAIL_EVALUATION, + guardrailName=guardrail_name, + guardrailDescription=description, + guardrailType="custom", + ) + with _active(span): + yield span + + +def set_validation_result(span: Span, result: GuardrailValidationResult) -> None: + """Record ``result``'s outcome on its evaluation span.""" + span.set_attribute("validationResult", result.result.name) + if result.reason: + span.set_attribute("details", [result.reason]) diff --git a/src/uipath_langchain/guardrails/middlewares/deterministic.py b/src/uipath_langchain/guardrails/middlewares/deterministic.py index 4f7151e53..9a65af3d2 100644 --- a/src/uipath_langchain/guardrails/middlewares/deterministic.py +++ b/src/uipath_langchain/guardrails/middlewares/deterministic.py @@ -21,7 +21,12 @@ from ..enums import GuardrailExecutionStage from ..models import GuardrailAction -from ._spans import guardrail_action_span +from ._spans import ( + guardrail_action_span, + guardrail_evaluation_span, + set_validation_result, + span_guid, +) from ._utils import ( convert_block_exception, create_modified_tool_request, @@ -235,18 +240,16 @@ def _evaluate_and_apply_modifications( ToolCallRequest | None, dict[str, Any] | None, ToolMessage | Command[Any] | None ]: """Evaluate rules and apply modifications if validation fails.""" - if not rules: - result = GuardrailValidationResult( - result=GuardrailValidationResultType.VALIDATION_FAILED, - reason="Empty rules - always apply action", - ) - else: + with guardrail_evaluation_span(self._name, self._description) as span: result = self._evaluate_rules(rules, stage, input_data, output_data) - if result.result != GuardrailValidationResultType.VALIDATION_FAILED: - return request, input_data, tool_result + set_validation_result(span, result) + if result.result != GuardrailValidationResultType.VALIDATION_FAILED: + return request, input_data, tool_result try: - modified_data = self._handle_validation_result(result, data_for_action) + modified_data = self._handle_validation_result( + result, data_for_action, evaluation_span_id=span_guid(span) + ) except GuardrailBlockException as exc: raise convert_block_exception(exc) from exc except AgentRuntimeError: @@ -383,10 +386,13 @@ def _extract_tool_output_data( return {"output": content} def _handle_validation_result( - self, result: GuardrailValidationResult, input_data: str | dict[str, Any] + self, + result: GuardrailValidationResult, + input_data: str | dict[str, Any], + evaluation_span_id: str | None = None, ) -> str | dict[str, Any] | None: """Handle guardrail validation result.""" if result.result != GuardrailValidationResultType.VALIDATION_FAILED: return None - with guardrail_action_span(self._name, self.action, result): + with guardrail_action_span(self._name, self.action, result, evaluation_span_id): return self.action.handle_validation_result(result, input_data, self._name) diff --git a/tests/guardrails/middlewares/test_deterministic_spans.py b/tests/guardrails/middlewares/test_deterministic_spans.py new file mode 100644 index 000000000..bfc346d57 --- /dev/null +++ b/tests/guardrails/middlewares/test_deterministic_spans.py @@ -0,0 +1,101 @@ +"""Deterministic rules record their check as a guardrailEvaluation span.""" + +from collections.abc import Iterator +from typing import Any +from unittest.mock import MagicMock + +import pytest +from langchain_core.messages import ToolMessage +from langgraph.prebuilt.tool_node import ToolCallRequest +from opentelemetry import trace +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry.trace import StatusCode +from uipath.platform.guardrails.decorators import BlockAction, LogAction + +from uipath_langchain.agent.exceptions import AgentRuntimeError +from uipath_langchain.guardrails import GuardrailAction, GuardrailExecutionStage +from uipath_langchain.guardrails.middlewares import ( + UiPathDeterministicGuardrailMiddleware, +) + + +@pytest.fixture +def spans(monkeypatch: pytest.MonkeyPatch) -> Iterator[InMemorySpanExporter]: + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + monkeypatch.setattr(trace, "get_tracer", provider.get_tracer) + yield exporter + provider.shutdown() + + +async def _call_weather(action: GuardrailAction, city: str) -> None: + rule = UiPathDeterministicGuardrailMiddleware( + tools=["get_weather"], + rules=[lambda args: args.get("city") == "London"], + action=action, + stage=GuardrailExecutionStage.PRE, + name="London Rule", + description="Flags London", + ) + [wrapper] = list(rule) + request = ToolCallRequest( + tool_call={"id": "tc1", "name": "get_weather", "args": {"city": city}}, + tool=MagicMock(), + state={}, + runtime=MagicMock(), + ) + + async def tool(request: ToolCallRequest) -> ToolMessage: + return ToolMessage(content="sunny", tool_call_id="tc1") + + await wrapper.awrap_tool_call(request, tool) + + +def _named(spans: InMemorySpanExporter, name: str) -> ReadableSpan: + [span] = [s for s in spans.get_finished_spans() if s.name == name] + return span + + +def _attributes(span: ReadableSpan) -> dict[str, Any]: + return dict(span.attributes or {}) + + +async def test_a_failed_rule_records_its_evaluation_and_links_the_action( + spans: InMemorySpanExporter, +) -> None: + await _call_weather(LogAction(), "London") + + evaluation = _attributes(_named(spans, "London Rule")) + assert evaluation["span_type"] == "guardrailEvaluation" + assert evaluation["guardrailType"] == "custom" + assert evaluation["guardrailDescription"] == "Flags London" + assert evaluation["validationResult"] == "VALIDATION_FAILED" + assert evaluation["details"] == ("Rule 1 detected violation",) + span_id = format(_named(spans, "London Rule").context.span_id, "016x") + log = _attributes(_named(spans, "Log")) + assert log["guardrailSpanId"] == f"00000000-0000-0000-{span_id[:4]}-{span_id[4:]}" + + +async def test_a_passing_rule_records_its_evaluation_only( + spans: InMemorySpanExporter, +) -> None: + await _call_weather(LogAction(), "Paris") + + evaluation = _attributes(_named(spans, "London Rule")) + assert evaluation["validationResult"] == "PASSED" + assert [s.name for s in spans.get_finished_spans()] == ["London Rule"] + + +async def test_a_blocking_rule_marks_its_action_as_failed( + spans: InMemorySpanExporter, +) -> None: + with pytest.raises(AgentRuntimeError): + await _call_weather(BlockAction(), "London") + + assert _named(spans, "Block").status.status_code == StatusCode.ERROR + assert _named(spans, "London Rule").status.status_code == StatusCode.OK diff --git a/uv.lock b/uv.lock index 445f0eba3..4267b7d4e 100644 --- a/uv.lock +++ b/uv.lock @@ -5067,7 +5067,7 @@ wheels = [ [[package]] name = "uipath-langchain" -version = "0.20.6" +version = "0.20.7" source = { editable = "." } dependencies = [ { name = "a2a-sdk" },