From 6fa85ecb03edf55c812fedaed6d9f3932cd7262e Mon Sep 17 00:00:00 2001 From: Daniel Joica Date: Thu, 19 Nov 2026 14:52:03 +0200 Subject: [PATCH 1/3] feat(guardrails): record deterministic rule checks as spans Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 2 +- .../guardrails/middlewares/_spans.py | 30 ++++++ .../guardrails/middlewares/deterministic.py | 48 ++++++--- .../middlewares/test_deterministic_spans.py | 101 ++++++++++++++++++ uv.lock | 2 +- 5 files changed, 165 insertions(+), 18 deletions(-) create mode 100644 tests/guardrails/middlewares/test_deterministic_spans.py diff --git a/pyproject.toml b/pyproject.toml index 7cceb2e7f..518e0c615 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath-langchain" -version = "0.20.4" +version = "0.20.5" description = "Python SDK that enables developers to build and deploy LangGraph agents to the UiPath Cloud Platform" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/src/uipath_langchain/guardrails/middlewares/_spans.py b/src/uipath_langchain/guardrails/middlewares/_spans.py index c623c25d6..f3a538c7c 100644 --- a/src/uipath_langchain/guardrails/middlewares/_spans.py +++ b/src/uipath_langchain/guardrails/middlewares/_spans.py @@ -13,6 +13,7 @@ from uipath.platform.guardrails.decorators._exceptions import GuardrailBlockException GUARDRAIL_ACTION = "guardrailAction" +GUARDRAIL_EVALUATION = "guardrailEvaluation" def _new_span(name: str, span_type: str, **attributes: Any) -> Span: @@ -52,6 +53,12 @@ def _error_text(exc: BaseException) -> str: return str(exc) +def span_guid(span: Span) -> str: + """Span id in the GUID form LLMOps uses.""" + span_id = format(span.get_span_context().span_id, "016x") + return f"00000000-0000-0000-{span_id[:4]}-{span_id[4:]}" + + def action_name(action: Any) -> str: """``BlockAction`` -> ``Block``.""" name = type(action).__name__ @@ -79,3 +86,26 @@ def guardrail_action_span( ) with _active(span): yield span + + +@contextmanager +def guardrail_evaluation_span( + guardrail_name: str, description: str | None = None +) -> Generator[Span]: + """Record a client-side check; set its outcome with ``set_validation_result``.""" + span = _new_span( + guardrail_name, + GUARDRAIL_EVALUATION, + guardrailName=guardrail_name, + guardrailDescription=description, + guardrailType="custom", + ) + with _active(span): + yield span + + +def set_validation_result(span: Span, result: GuardrailValidationResult) -> None: + """Record ``result`` the way the Guardrails Service does.""" + span.set_attribute("validationResult", result.result.name) + if result.reason: + span.set_attribute("details", [result.reason]) diff --git a/src/uipath_langchain/guardrails/middlewares/deterministic.py b/src/uipath_langchain/guardrails/middlewares/deterministic.py index 4f7151e53..d6f0a3c30 100644 --- a/src/uipath_langchain/guardrails/middlewares/deterministic.py +++ b/src/uipath_langchain/guardrails/middlewares/deterministic.py @@ -17,11 +17,14 @@ ) from uipath.platform.guardrails.decorators._exceptions import GuardrailBlockException -from uipath_langchain.agent.exceptions import AgentRuntimeError - from ..enums import GuardrailExecutionStage from ..models import GuardrailAction -from ._spans import guardrail_action_span +from ._spans import ( + guardrail_action_span, + guardrail_evaluation_span, + set_validation_result, + span_guid, +) from ._utils import ( convert_block_exception, create_modified_tool_request, @@ -235,22 +238,14 @@ def _evaluate_and_apply_modifications( ToolCallRequest | None, dict[str, Any] | None, ToolMessage | Command[Any] | None ]: """Evaluate rules and apply modifications if validation fails.""" - if not rules: - result = GuardrailValidationResult( - result=GuardrailValidationResultType.VALIDATION_FAILED, - reason="Empty rules - always apply action", - ) - else: - result = self._evaluate_rules(rules, stage, input_data, output_data) - if result.result != GuardrailValidationResultType.VALIDATION_FAILED: - return request, input_data, tool_result - try: - modified_data = self._handle_validation_result(result, data_for_action) + failed, modified_data = self._check( + stage, rules, input_data, output_data, data_for_action + ) except GuardrailBlockException as exc: raise convert_block_exception(exc) from exc - except AgentRuntimeError: - raise + if not failed: + return request, input_data, tool_result if stage == GuardrailExecutionStage.PRE: if modified_data is not None and isinstance(modified_data, dict): @@ -269,6 +264,27 @@ def _evaluate_and_apply_modifications( return request, input_data, tool_result + def _check( + self, + stage: GuardrailExecutionStage, + rules: Sequence[RuleFunction], + input_data: dict[str, Any] | None, + output_data: dict[str, Any] | None, + data_for_action: dict[str, Any] | str, + ) -> tuple[bool, str | dict[str, Any] | None]: + """Evaluate the rules and apply the action on a violation. + + Returns: + Whether the rules failed, and the data the action modified. + """ + with guardrail_evaluation_span(self._name, self._description) as span: + result = self._evaluate_rules(rules, stage, input_data, output_data) + set_validation_result(span, result) + if result.result != GuardrailValidationResultType.VALIDATION_FAILED: + return False, None + result = result.model_copy(update={"span_id": span_guid(span)}) + return True, self._handle_validation_result(result, data_for_action) + def _evaluate_rules( self, rules: Sequence[RuleFunction], diff --git a/tests/guardrails/middlewares/test_deterministic_spans.py b/tests/guardrails/middlewares/test_deterministic_spans.py new file mode 100644 index 000000000..bfc346d57 --- /dev/null +++ b/tests/guardrails/middlewares/test_deterministic_spans.py @@ -0,0 +1,101 @@ +"""Deterministic rules record their check as a guardrailEvaluation span.""" + +from collections.abc import Iterator +from typing import Any +from unittest.mock import MagicMock + +import pytest +from langchain_core.messages import ToolMessage +from langgraph.prebuilt.tool_node import ToolCallRequest +from opentelemetry import trace +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry.trace import StatusCode +from uipath.platform.guardrails.decorators import BlockAction, LogAction + +from uipath_langchain.agent.exceptions import AgentRuntimeError +from uipath_langchain.guardrails import GuardrailAction, GuardrailExecutionStage +from uipath_langchain.guardrails.middlewares import ( + UiPathDeterministicGuardrailMiddleware, +) + + +@pytest.fixture +def spans(monkeypatch: pytest.MonkeyPatch) -> Iterator[InMemorySpanExporter]: + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + monkeypatch.setattr(trace, "get_tracer", provider.get_tracer) + yield exporter + provider.shutdown() + + +async def _call_weather(action: GuardrailAction, city: str) -> None: + rule = UiPathDeterministicGuardrailMiddleware( + tools=["get_weather"], + rules=[lambda args: args.get("city") == "London"], + action=action, + stage=GuardrailExecutionStage.PRE, + name="London Rule", + description="Flags London", + ) + [wrapper] = list(rule) + request = ToolCallRequest( + tool_call={"id": "tc1", "name": "get_weather", "args": {"city": city}}, + tool=MagicMock(), + state={}, + runtime=MagicMock(), + ) + + async def tool(request: ToolCallRequest) -> ToolMessage: + return ToolMessage(content="sunny", tool_call_id="tc1") + + await wrapper.awrap_tool_call(request, tool) + + +def _named(spans: InMemorySpanExporter, name: str) -> ReadableSpan: + [span] = [s for s in spans.get_finished_spans() if s.name == name] + return span + + +def _attributes(span: ReadableSpan) -> dict[str, Any]: + return dict(span.attributes or {}) + + +async def test_a_failed_rule_records_its_evaluation_and_links_the_action( + spans: InMemorySpanExporter, +) -> None: + await _call_weather(LogAction(), "London") + + evaluation = _attributes(_named(spans, "London Rule")) + assert evaluation["span_type"] == "guardrailEvaluation" + assert evaluation["guardrailType"] == "custom" + assert evaluation["guardrailDescription"] == "Flags London" + assert evaluation["validationResult"] == "VALIDATION_FAILED" + assert evaluation["details"] == ("Rule 1 detected violation",) + span_id = format(_named(spans, "London Rule").context.span_id, "016x") + log = _attributes(_named(spans, "Log")) + assert log["guardrailSpanId"] == f"00000000-0000-0000-{span_id[:4]}-{span_id[4:]}" + + +async def test_a_passing_rule_records_its_evaluation_only( + spans: InMemorySpanExporter, +) -> None: + await _call_weather(LogAction(), "Paris") + + evaluation = _attributes(_named(spans, "London Rule")) + assert evaluation["validationResult"] == "PASSED" + assert [s.name for s in spans.get_finished_spans()] == ["London Rule"] + + +async def test_a_blocking_rule_marks_its_action_as_failed( + spans: InMemorySpanExporter, +) -> None: + with pytest.raises(AgentRuntimeError): + await _call_weather(BlockAction(), "London") + + assert _named(spans, "Block").status.status_code == StatusCode.ERROR + assert _named(spans, "London Rule").status.status_code == StatusCode.OK diff --git a/uv.lock b/uv.lock index a437877d2..c60fd90ce 100644 --- a/uv.lock +++ b/uv.lock @@ -4861,7 +4861,7 @@ wheels = [ [[package]] name = "uipath-langchain" -version = "0.20.4" +version = "0.20.5" source = { editable = "." } dependencies = [ { name = "a2a-sdk" }, From d116b5d08053da70f7bf94f79b64a9db94ef42f8 Mon Sep 17 00:00:00 2001 From: Daniel Joica Date: Thu, 19 Nov 2026 15:22:03 +0200 Subject: [PATCH 2/3] refactor(guardrails): pass the check's span id to the action span directly Co-Authored-By: Claude Opus 5.5 --- .../guardrails/middlewares/_spans.py | 19 ++++++-- .../guardrails/middlewares/deterministic.py | 44 +++++++------------ 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/src/uipath_langchain/guardrails/middlewares/_spans.py b/src/uipath_langchain/guardrails/middlewares/_spans.py index f3a538c7c..7373e82ea 100644 --- a/src/uipath_langchain/guardrails/middlewares/_spans.py +++ b/src/uipath_langchain/guardrails/middlewares/_spans.py @@ -54,7 +54,11 @@ def _error_text(exc: BaseException) -> str: def span_guid(span: Span) -> str: - """Span id in the GUID form LLMOps uses.""" + """``span``'s id as a GUID, the form the Guardrails Service returns its span ids in. + + Action spans link to their check through ``guardrailSpanId``; this keeps + that link in one format whether the check ran in the service or locally. + """ span_id = format(span.get_span_context().span_id, "016x") return f"00000000-0000-0000-{span_id[:4]}-{span_id[4:]}" @@ -67,16 +71,23 @@ def action_name(action: Any) -> str: @contextmanager def guardrail_action_span( - guardrail_name: str, action: Any, result: GuardrailValidationResult + guardrail_name: str, + action: Any, + result: GuardrailValidationResult, + evaluation_span_id: str | None = None, ) -> Generator[Span]: - """Record the action taken on a failed ``result``.""" + """Record the action taken on a failed ``result``. + + The span links to the check that failed: ``evaluation_span_id`` for a + check recorded locally, otherwise the service's span id on ``result``. + """ severity = getattr(action, "severity_level", None) span = _new_span( action_name(action), GUARDRAIL_ACTION, guardrailName=guardrail_name, action=action_name(action), - guardrailSpanId=result.span_id, + guardrailSpanId=evaluation_span_id or result.span_id, reason=result.reason or None, severityLevel=( severity.name.title() diff --git a/src/uipath_langchain/guardrails/middlewares/deterministic.py b/src/uipath_langchain/guardrails/middlewares/deterministic.py index d6f0a3c30..9a65af3d2 100644 --- a/src/uipath_langchain/guardrails/middlewares/deterministic.py +++ b/src/uipath_langchain/guardrails/middlewares/deterministic.py @@ -17,6 +17,8 @@ ) from uipath.platform.guardrails.decorators._exceptions import GuardrailBlockException +from uipath_langchain.agent.exceptions import AgentRuntimeError + from ..enums import GuardrailExecutionStage from ..models import GuardrailAction from ._spans import ( @@ -238,14 +240,20 @@ def _evaluate_and_apply_modifications( ToolCallRequest | None, dict[str, Any] | None, ToolMessage | Command[Any] | None ]: """Evaluate rules and apply modifications if validation fails.""" + with guardrail_evaluation_span(self._name, self._description) as span: + result = self._evaluate_rules(rules, stage, input_data, output_data) + set_validation_result(span, result) + if result.result != GuardrailValidationResultType.VALIDATION_FAILED: + return request, input_data, tool_result + try: - failed, modified_data = self._check( - stage, rules, input_data, output_data, data_for_action + modified_data = self._handle_validation_result( + result, data_for_action, evaluation_span_id=span_guid(span) ) except GuardrailBlockException as exc: raise convert_block_exception(exc) from exc - if not failed: - return request, input_data, tool_result + except AgentRuntimeError: + raise if stage == GuardrailExecutionStage.PRE: if modified_data is not None and isinstance(modified_data, dict): @@ -264,27 +272,6 @@ def _evaluate_and_apply_modifications( return request, input_data, tool_result - def _check( - self, - stage: GuardrailExecutionStage, - rules: Sequence[RuleFunction], - input_data: dict[str, Any] | None, - output_data: dict[str, Any] | None, - data_for_action: dict[str, Any] | str, - ) -> tuple[bool, str | dict[str, Any] | None]: - """Evaluate the rules and apply the action on a violation. - - Returns: - Whether the rules failed, and the data the action modified. - """ - with guardrail_evaluation_span(self._name, self._description) as span: - result = self._evaluate_rules(rules, stage, input_data, output_data) - set_validation_result(span, result) - if result.result != GuardrailValidationResultType.VALIDATION_FAILED: - return False, None - result = result.model_copy(update={"span_id": span_guid(span)}) - return True, self._handle_validation_result(result, data_for_action) - def _evaluate_rules( self, rules: Sequence[RuleFunction], @@ -399,10 +386,13 @@ def _extract_tool_output_data( return {"output": content} def _handle_validation_result( - self, result: GuardrailValidationResult, input_data: str | dict[str, Any] + self, + result: GuardrailValidationResult, + input_data: str | dict[str, Any], + evaluation_span_id: str | None = None, ) -> str | dict[str, Any] | None: """Handle guardrail validation result.""" if result.result != GuardrailValidationResultType.VALIDATION_FAILED: return None - with guardrail_action_span(self._name, self.action, result): + with guardrail_action_span(self._name, self.action, result, evaluation_span_id): return self.action.handle_validation_result(result, input_data, self._name) From 8464d4254e6af9e55fbda192427770bbbc42a983 Mon Sep 17 00:00:00 2001 From: Daniel Joica Date: Thu, 19 Nov 2026 15:44:14 +0200 Subject: [PATCH 3/3] docs(guardrails): shorten the span helpers' docstrings Co-Authored-By: Claude Opus 5.5 --- .../guardrails/middlewares/_spans.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/uipath_langchain/guardrails/middlewares/_spans.py b/src/uipath_langchain/guardrails/middlewares/_spans.py index 7373e82ea..36c8eba19 100644 --- a/src/uipath_langchain/guardrails/middlewares/_spans.py +++ b/src/uipath_langchain/guardrails/middlewares/_spans.py @@ -54,11 +54,7 @@ def _error_text(exc: BaseException) -> str: def span_guid(span: Span) -> str: - """``span``'s id as a GUID, the form the Guardrails Service returns its span ids in. - - Action spans link to their check through ``guardrailSpanId``; this keeps - that link in one format whether the check ran in the service or locally. - """ + """``span``'s id as a GUID, the form ``guardrailSpanId`` uses.""" span_id = format(span.get_span_context().span_id, "016x") return f"00000000-0000-0000-{span_id[:4]}-{span_id[4:]}" @@ -76,11 +72,7 @@ def guardrail_action_span( result: GuardrailValidationResult, evaluation_span_id: str | None = None, ) -> Generator[Span]: - """Record the action taken on a failed ``result``. - - The span links to the check that failed: ``evaluation_span_id`` for a - check recorded locally, otherwise the service's span id on ``result``. - """ + """Record the action taken on a failed ``result``.""" severity = getattr(action, "severity_level", None) span = _new_span( action_name(action), @@ -103,7 +95,7 @@ def guardrail_action_span( def guardrail_evaluation_span( guardrail_name: str, description: str | None = None ) -> Generator[Span]: - """Record a client-side check; set its outcome with ``set_validation_result``.""" + """Record a rule check that runs in the agent.""" span = _new_span( guardrail_name, GUARDRAIL_EVALUATION, @@ -116,7 +108,7 @@ def guardrail_evaluation_span( def set_validation_result(span: Span, result: GuardrailValidationResult) -> None: - """Record ``result`` the way the Guardrails Service does.""" + """Record ``result``'s outcome on its evaluation span.""" span.set_attribute("validationResult", result.result.name) if result.reason: span.set_attribute("details", [result.reason])