From 0086f380f99b3ee3b4d7c4eea128274e63d9c7b1 Mon Sep 17 00:00:00 2001 From: Andrei Petraru Date: Sat, 24 Oct 2026 23:19:52 +0300 Subject: [PATCH] feat(guardrails): report provider errors on guardrail validation results [AL-673] The guardrail validate API answers PROVIDER_ERROR, INTERNAL_ERROR or RATE_LIMIT_EXCEEDED when it could not evaluate the input, for example when an LLM-as-Judge model rejects the request. evaluate_guardrail turned these into VALIDATION_FAILED through its unknown-result fallback, so callers could not tell a real violation from a failed evaluation (SRE-687087: judge models that deprecate `temperature` answered 400 and 90 eval runs were blocked). GuardrailValidationResult gains an additive error_type (GuardrailValidationErrorType) that is set for those results. result stays validation_failed, so every existing VALIDATION_FAILED check keeps firing its action (fail closed); error_type only says why the validation did not complete. Error and unknown results are now logged as warnings. Bumps uipath-core to 0.5.34 and uipath-platform to 0.2.42 (requiring uipath-core>=0.5.34) and refreshes the uv.lock files. Co-Authored-By: Claude Opus 5.5 --- packages/uipath-core/pyproject.toml | 2 +- .../src/uipath/core/guardrails/__init__.py | 2 + .../src/uipath/core/guardrails/guardrails.py | 19 +++ .../guardrails/test_guardrails_models.py | 55 +++++++++ packages/uipath-core/uv.lock | 2 +- packages/uipath-platform/pyproject.toml | 4 +- .../guardrails/_guardrails_service.py | 46 +++++++- .../tests/services/test_guardrails_service.py | 111 ++++++++++++++++++ packages/uipath-platform/uv.lock | 4 +- packages/uipath/uv.lock | 4 +- 10 files changed, 239 insertions(+), 10 deletions(-) diff --git a/packages/uipath-core/pyproject.toml b/packages/uipath-core/pyproject.toml index 80e2327f8..1d328bd86 100644 --- a/packages/uipath-core/pyproject.toml +++ b/packages/uipath-core/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath-core" -version = "0.5.33" +version = "0.5.34" description = "UiPath Core abstractions" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/packages/uipath-core/src/uipath/core/guardrails/__init__.py b/packages/uipath-core/src/uipath/core/guardrails/__init__.py index 4ade20740..4953a1c73 100644 --- a/packages/uipath-core/src/uipath/core/guardrails/__init__.py +++ b/packages/uipath-core/src/uipath/core/guardrails/__init__.py @@ -15,6 +15,7 @@ FieldSource, GuardrailScope, GuardrailSelector, + GuardrailValidationErrorType, GuardrailValidationResult, GuardrailValidationResultType, NumberRule, @@ -45,4 +46,5 @@ "GuardrailSelector", "GuardrailValidationResult", "GuardrailValidationResultType", + "GuardrailValidationErrorType", ] diff --git a/packages/uipath-core/src/uipath/core/guardrails/guardrails.py b/packages/uipath-core/src/uipath/core/guardrails/guardrails.py index bd218c4fb..19a7ef351 100644 --- a/packages/uipath-core/src/uipath/core/guardrails/guardrails.py +++ b/packages/uipath-core/src/uipath/core/guardrails/guardrails.py @@ -24,9 +24,21 @@ class GuardrailValidationResultType(str, Enum): FEATURE_DISABLED = "feature_disabled" +class GuardrailValidationErrorType(str, Enum): + """Why the guardrail service could not complete a validation.""" + + PROVIDER_ERROR = "provider_error" + INTERNAL_ERROR = "internal_error" + RATE_LIMIT_EXCEEDED = "rate_limit_exceeded" + + class GuardrailValidationResult(BaseModel): """Result returned from validating input with a given guardrail. + When the guardrail service could not evaluate the input, ``result`` stays + ``validation_failed`` so the guardrail action still fires (fail closed), and + ``error_type`` says why the validation did not complete. + Attributes: result: The validation result type. reason: Textual explanation describing why the validation passed or failed. @@ -34,6 +46,8 @@ class GuardrailValidationResult(BaseModel): for trace correlation. None when the response omits the header. flagged_attachment_ids: Ids of the attachments the guardrail service flagged as causing the violation. None when the response omits them. + error_type: Why the guardrail service could not complete the validation. + None for every other result, including an unrecognised one. """ model_config = ConfigDict(populate_by_name=True) @@ -54,6 +68,11 @@ class GuardrailValidationResult(BaseModel): alias="flaggedAttachmentIds", description="Ids of the attachments flagged by the guardrail service.", ) + error_type: Optional[GuardrailValidationErrorType] = Field( + default=None, + alias="errorType", + description="Why the guardrail service could not complete the validation.", + ) class FieldSource(str, Enum): diff --git a/packages/uipath-core/tests/guardrails/test_guardrails_models.py b/packages/uipath-core/tests/guardrails/test_guardrails_models.py index 0744a8db6..45c58429d 100644 --- a/packages/uipath-core/tests/guardrails/test_guardrails_models.py +++ b/packages/uipath-core/tests/guardrails/test_guardrails_models.py @@ -6,6 +6,7 @@ AllFieldsSelector, FieldReference, FieldSource, + GuardrailValidationErrorType, GuardrailValidationResult, GuardrailValidationResultType, ) @@ -103,3 +104,57 @@ def test_dumps_none_by_alias_like_span_id(self) -> None: assert "flaggedAttachmentIds" not in result.model_dump( by_alias=True, exclude_none=True ) + + +class TestGuardrailValidationResultErrorType: + """Test GuardrailValidationResult.error_type.""" + + def test_enum_values(self) -> None: + assert GuardrailValidationErrorType.PROVIDER_ERROR.value == "provider_error" + assert GuardrailValidationErrorType.INTERNAL_ERROR.value == "internal_error" + assert ( + GuardrailValidationErrorType.RATE_LIMIT_EXCEEDED.value + == "rate_limit_exceeded" + ) + + def test_defaults_to_none(self) -> None: + result = GuardrailValidationResult( + result=GuardrailValidationResultType.VALIDATION_FAILED, reason="" + ) + assert result.error_type is None + + def test_populates_from_alias(self) -> None: + result = GuardrailValidationResult.model_validate( + { + "result": "validation_failed", + "reason": "Judge model rejected the request", + "errorType": "provider_error", + } + ) + assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.error_type == GuardrailValidationErrorType.PROVIDER_ERROR + + def test_populates_from_field_name(self) -> None: + result = GuardrailValidationResult( + result=GuardrailValidationResultType.VALIDATION_FAILED, + reason="", + error_type=GuardrailValidationErrorType.RATE_LIMIT_EXCEEDED, + ) + assert result.error_type == GuardrailValidationErrorType.RATE_LIMIT_EXCEEDED + + def test_round_trips_by_alias(self) -> None: + result = GuardrailValidationResult( + result=GuardrailValidationResultType.VALIDATION_FAILED, + reason="", + error_type=GuardrailValidationErrorType.INTERNAL_ERROR, + ) + dumped = result.model_dump(by_alias=True) + assert dumped["errorType"] == GuardrailValidationErrorType.INTERNAL_ERROR + assert GuardrailValidationResult.model_validate(dumped) == result + + def test_dumps_none_by_alias(self) -> None: + result = GuardrailValidationResult( + result=GuardrailValidationResultType.PASSED, reason="" + ) + assert result.model_dump(by_alias=True)["errorType"] is None + assert "errorType" not in result.model_dump(by_alias=True, exclude_none=True) diff --git a/packages/uipath-core/uv.lock b/packages/uipath-core/uv.lock index 8cc930b64..48fa850ef 100644 --- a/packages/uipath-core/uv.lock +++ b/packages/uipath-core/uv.lock @@ -1011,7 +1011,7 @@ wheels = [ [[package]] name = "uipath-core" -version = "0.5.33" +version = "0.5.34" source = { editable = "." } dependencies = [ { name = "opentelemetry-instrumentation" }, diff --git a/packages/uipath-platform/pyproject.toml b/packages/uipath-platform/pyproject.toml index c3e51954e..7d786f3d3 100644 --- a/packages/uipath-platform/pyproject.toml +++ b/packages/uipath-platform/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath-platform" -version = "0.2.41" +version = "0.2.42" description = "HTTP client library for programmatic access to UiPath Platform" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" @@ -9,7 +9,7 @@ dependencies = [ "httpx>=0.28.1", "tenacity>=9.0.0", "truststore>=0.10.1", - "uipath-core>=0.5.33, <0.6.0", + "uipath-core>=0.5.34, <0.6.0", "pydantic-function-models>=0.1.11", "sqlparse>=0.6.0, <0.7.0", ] diff --git a/packages/uipath-platform/src/uipath/platform/guardrails/_guardrails_service.py b/packages/uipath-platform/src/uipath/platform/guardrails/_guardrails_service.py index f3b87b8dd..2af4f0e87 100644 --- a/packages/uipath-platform/src/uipath/platform/guardrails/_guardrails_service.py +++ b/packages/uipath-platform/src/uipath/platform/guardrails/_guardrails_service.py @@ -1,8 +1,10 @@ +import logging import re from typing import Any, Optional from httpx import HTTPStatusError from uipath.core.guardrails import ( + GuardrailValidationErrorType, GuardrailValidationResult, GuardrailValidationResultType, ) @@ -24,6 +26,8 @@ GuardrailTerminationMode, ) +logger = logging.getLogger(__name__) + _ATTACHMENT_VALIDATE_TIMEOUT_SECONDS = 60.0 @@ -103,8 +107,31 @@ def _parse_result(result_str: str) -> GuardrailValidationResultType: return GuardrailValidationResultType[result_str] except KeyError: # Fallback to validation_failed if unknown + logger.warning( + "Unknown guardrail validation result '%s'; treating it as %s.", + result_str, + GuardrailValidationResultType.VALIDATION_FAILED.value, + ) return GuardrailValidationResultType.VALIDATION_FAILED + @staticmethod + def _parse_error_type(result_str: Any) -> Optional[GuardrailValidationErrorType]: + """Parse a result string the API returns when it could not validate. + + Args: + result_str: The result string from the API response (e.g., "PROVIDER_ERROR"). + + Returns: + Optional[GuardrailValidationErrorType]: Why the validation could not + complete, or None when the result is not an error. + """ + if not isinstance(result_str, str): + return None + try: + return GuardrailValidationErrorType(result_str.lower()) + except ValueError: + return None + @traced("evaluate_guardrail", run_type="uipath") def evaluate_guardrail( self, @@ -220,15 +247,30 @@ def evaluate_guardrail( else: raise - result = self._parse_result(response_data.get("result", "")) - + result_str = response_data.get("result", "") reason = response_data.get("details", "") + error_type = self._parse_error_type(result_str) + if error_type is not None: + # The service could not evaluate the input: fail closed so the + # guardrail action still fires, and report why on error_type. + logger.warning( + "Guardrail '%s' could not be evaluated (%s): %s", + guardrail.name, + result_str, + reason, + ) + result = GuardrailValidationResultType.VALIDATION_FAILED + else: + result = self._parse_result(result_str) + # Prepare model data model_data: dict[str, Any] = { "result": result.value, "reason": reason, } + if error_type is not None: + model_data["errorType"] = error_type.value if span_id: model_data["spanId"] = span_id flagged_attachment_ids = response_data.get("flaggedAttachmentIds") diff --git a/packages/uipath-platform/tests/services/test_guardrails_service.py b/packages/uipath-platform/tests/services/test_guardrails_service.py index d135c5c66..652b628fc 100644 --- a/packages/uipath-platform/tests/services/test_guardrails_service.py +++ b/packages/uipath-platform/tests/services/test_guardrails_service.py @@ -1,4 +1,5 @@ import json +import logging import httpx import pytest @@ -6,6 +7,7 @@ from uipath.core.guardrails import ( GuardrailScope, GuardrailSelector, + GuardrailValidationErrorType, GuardrailValidationResultType, ) @@ -1378,4 +1380,113 @@ def test_unknown_response_keys_are_ignored( "reason": "PII detected", "spanId": None, "flaggedAttachmentIds": [_ATTACHMENT_ID], + "errorType": None, } + + +_SERVICE_LOGGER = "uipath.platform.guardrails._guardrails_service" + + +class TestGuardrailErrorType: + """evaluate_guardrail fails closed and reports why validation did not complete.""" + + @pytest.mark.parametrize( + ("result_value", "expected"), + [ + ("PROVIDER_ERROR", GuardrailValidationErrorType.PROVIDER_ERROR), + ("INTERNAL_ERROR", GuardrailValidationErrorType.INTERNAL_ERROR), + ( + "RATE_LIMIT_EXCEEDED", + GuardrailValidationErrorType.RATE_LIMIT_EXCEEDED, + ), + ("provider_error", GuardrailValidationErrorType.PROVIDER_ERROR), + ], + ) + def test_error_results_fail_closed_with_error_type( + self, + httpx_mock: HTTPXMock, + service: GuardrailsService, + base_url: str, + org: str, + tenant: str, + caplog: pytest.LogCaptureFixture, + result_value: str, + expected: GuardrailValidationErrorType, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}{_VALIDATE_PATH}", + status_code=200, + json={ + "result": result_value, + "details": "`temperature` is deprecated for this model.", + }, + ) + + with caplog.at_level(logging.WARNING, logger=_SERVICE_LOGGER): + result = service.evaluate_guardrail("x", _judge_guardrail()) + + assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.error_type == expected + assert result.reason == "`temperature` is deprecated for this model." + assert any( + record.levelno == logging.WARNING and result_value in record.getMessage() + for record in caplog.records + ) + + def test_unknown_result_fails_closed_without_error_type( + self, + httpx_mock: HTTPXMock, + service: GuardrailsService, + base_url: str, + org: str, + tenant: str, + caplog: pytest.LogCaptureFixture, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}{_VALIDATE_PATH}", + status_code=200, + json={"result": "SOMETHING_NEW", "details": "Unexpected"}, + ) + + with caplog.at_level(logging.WARNING, logger=_SERVICE_LOGGER): + result = service.evaluate_guardrail("x", _judge_guardrail()) + + assert result.result == GuardrailValidationResultType.VALIDATION_FAILED + assert result.error_type is None + assert result.reason == "Unexpected" + assert any( + record.levelno == logging.WARNING and "SOMETHING_NEW" in record.getMessage() + for record in caplog.records + ) + + @pytest.mark.parametrize( + ("result_value", "expected"), + [ + ("PASSED", GuardrailValidationResultType.PASSED), + ("VALIDATION_FAILED", GuardrailValidationResultType.VALIDATION_FAILED), + ], + ) + def test_evaluated_results_have_no_error_type( + self, + httpx_mock: HTTPXMock, + service: GuardrailsService, + base_url: str, + org: str, + tenant: str, + caplog: pytest.LogCaptureFixture, + result_value: str, + expected: GuardrailValidationResultType, + ) -> None: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}{_VALIDATE_PATH}", + status_code=200, + json={"result": result_value, "details": "Evaluated"}, + ) + + with caplog.at_level(logging.WARNING, logger=_SERVICE_LOGGER): + result = service.evaluate_guardrail("x", _judge_guardrail()) + + assert result.result == expected + assert result.error_type is None + assert result.reason == "Evaluated" + assert not [r for r in caplog.records if r.name == _SERVICE_LOGGER] diff --git a/packages/uipath-platform/uv.lock b/packages/uipath-platform/uv.lock index 5c59bdb8e..8bef3ab89 100644 --- a/packages/uipath-platform/uv.lock +++ b/packages/uipath-platform/uv.lock @@ -1063,7 +1063,7 @@ wheels = [ [[package]] name = "uipath-core" -version = "0.5.33" +version = "0.5.34" source = { editable = "../uipath-core" } dependencies = [ { name = "opentelemetry-instrumentation" }, @@ -1095,7 +1095,7 @@ dev = [ [[package]] name = "uipath-platform" -version = "0.2.41" +version = "0.2.42" source = { editable = "." } dependencies = [ { name = "anyio" }, diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index 3e9313fca..972c5380d 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -2721,7 +2721,7 @@ dev = [ [[package]] name = "uipath-core" -version = "0.5.33" +version = "0.5.34" source = { editable = "../uipath-core" } dependencies = [ { name = "opentelemetry-instrumentation" }, @@ -2762,7 +2762,7 @@ wheels = [ [[package]] name = "uipath-platform" -version = "0.2.41" +version = "0.2.42" source = { editable = "../uipath-platform" } dependencies = [ { name = "anyio" },