diff --git a/packages/uipath-platform/pyproject.toml b/packages/uipath-platform/pyproject.toml index 78f8bc068..fdafe30cd 100644 --- a/packages/uipath-platform/pyproject.toml +++ b/packages/uipath-platform/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath-platform" -version = "0.3.12" +version = "0.3.13" description = "HTTP client library for programmatic access to UiPath Platform" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/packages/uipath-platform/src/uipath/platform/_uipath.py b/packages/uipath-platform/src/uipath/platform/_uipath.py index dddbc2c37..e3b5836ad 100644 --- a/packages/uipath-platform/src/uipath/platform/_uipath.py +++ b/packages/uipath-platform/src/uipath/platform/_uipath.py @@ -139,7 +139,9 @@ def documents(self) -> DocumentsService: @property def documents_v2(self) -> DocumentsServiceV2: - return DocumentsServiceV2(self._config, self._execution_context) + return DocumentsServiceV2( + self._config, self._execution_context, folders_service=self.folders + ) @property def queues(self) -> QueuesService: diff --git a/packages/uipath-platform/src/uipath/platform/documents/__init__.py b/packages/uipath-platform/src/uipath/platform/documents/__init__.py index 0b522e182..86c35fa8c 100644 --- a/packages/uipath-platform/src/uipath/platform/documents/__init__.py +++ b/packages/uipath-platform/src/uipath/platform/documents/__init__.py @@ -17,10 +17,12 @@ FieldType, FieldValueProjection, FileContent, + ModelExtractionResponse, ProjectType, Reference, StartExtractionResponse, StartExtractionValidationResponse, + StartModelExtractionResponse, StartOperationResponse, ValidateClassificationAction, ValidateExtractionAction, @@ -49,4 +51,6 @@ "StartExtractionResponse", "StartOperationResponse", "StartExtractionValidationResponse", + "StartModelExtractionResponse", + "ModelExtractionResponse", ] diff --git a/packages/uipath-platform/src/uipath/platform/documents/_documents_service.py b/packages/uipath-platform/src/uipath/platform/documents/_documents_service.py index 0357ea20a..05a73151d 100644 --- a/packages/uipath-platform/src/uipath/platform/documents/_documents_service.py +++ b/packages/uipath-platform/src/uipath/platform/documents/_documents_service.py @@ -3,7 +3,18 @@ import time from contextlib import nullcontext from pathlib import Path -from typing import Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple, Union +from typing import ( + Any, + Awaitable, + Callable, + Dict, + List, + Optional, + Set, + Tuple, + TypeVar, + Union, +) from urllib.parse import quote from uuid import UUID @@ -33,6 +44,8 @@ POLLING_INTERVAL = 2 # seconds POLLING_TIMEOUT = 300 # seconds +T = TypeVar("T") + # Response values returned as-is: Document Processing contract payloads are # PascalCase in every API version, and the digitization DOM is large and unused. _UNNORMALIZED_KEYS = frozenset( @@ -46,15 +59,33 @@ ) -def _framework_url(*segments: Any) -> Endpoint: +def _url_path(segments: Tuple[Any, ...]) -> str: parts = [str(segment) for segment in segments] for part in parts: # Percent-encoding leaves these unchanged, and URL normalization would # resolve them, moving the request to another route. if part in ("", ".", ".."): raise ValueError(f"Invalid URL path segment: {part!r}") - path = "/".join(quote(part, safe="") for part in parts) - return Endpoint(f"/du_/api/framework/projects/{path}") + return "/".join(quote(part, safe="") for part in parts) + + +def _framework_url(*segments: Any) -> Endpoint: + return Endpoint(f"/du_/api/framework/projects/{_url_path(segments)}") + + +def _status_result( + outcome: Tuple[Any, Optional[Any], Optional[Any]], + wait_statuses: List[str], + success_status: str, +) -> Any: + """Read one `(status, error, result)` check for `_wait_until_complete`.""" + status, error, result = outcome + if status in wait_statuses: + # Only signals the poller to try again; it never reaches the caller. + raise OperationNotCompleteException(operation_id="", status=status) + if status != success_status: + raise RuntimeError(f"Operation failed with status: {status}, error: {error}") + return result def _operation_result( @@ -801,31 +832,47 @@ async def _start_extraction_async( api_version=self._api_version, ) + def _wait_until_complete(self, retrieve: Callable[[], T]) -> T: + """Call `retrieve` until it stops raising `OperationNotCompleteException`. + + This is the one polling loop of the documents services: blocking + methods wait through it, on the same single-check `retrieve` that + non-blocking callers use. + + Raises: + TimeoutError: If the operation is still not complete after `polling_timeout`. + """ + start_time = time.monotonic() + while True: + try: + return retrieve() + except OperationNotCompleteException: + if time.monotonic() - start_time >= self.polling_timeout: + raise TimeoutError("Operation timed out.") from None + time.sleep(self.polling_interval) + + async def _wait_until_complete_async( + self, retrieve: Callable[[], Awaitable[T]] + ) -> T: + """Asynchronous version of `_wait_until_complete`.""" + start_time = time.monotonic() + while True: + try: + return await retrieve() + except OperationNotCompleteException: + if time.monotonic() - start_time >= self.polling_timeout: + raise TimeoutError("Operation timed out.") from None + await asyncio.sleep(self.polling_interval) + def _wait_for_operation( self, result_getter: Callable[[], Tuple[Any, Optional[Any], Optional[Any]]], wait_statuses: List[str], success_status: str, ) -> Any: - start_time = time.monotonic() - status = wait_statuses[0] - result = None - - while ( - status in wait_statuses - and (time.monotonic() - start_time) < self.polling_timeout - ): - status, error, result = result_getter() - time.sleep(self.polling_interval) - - if status != success_status: - if time.monotonic() - start_time >= self.polling_timeout: - raise TimeoutError("Operation timed out.") - raise RuntimeError( - f"Operation failed with status: {status}, error: {error}" - ) - - return result + return self._wait_until_complete( + lambda: _status_result(result_getter(), wait_statuses, success_status) + ) async def _wait_for_operation_async( self, @@ -835,25 +882,10 @@ async def _wait_for_operation_async( wait_statuses: List[str], success_status: str, ) -> Any: - start_time = time.monotonic() - status = wait_statuses[0] - result = None - - while ( - status in wait_statuses - and (time.monotonic() - start_time) < self.polling_timeout - ): - status, error, result = await result_getter() - await asyncio.sleep(self.polling_interval) - - if status != success_status: - if time.monotonic() - start_time >= self.polling_timeout: - raise TimeoutError("Operation timed out.") - raise RuntimeError( - f"Operation failed with status: {status}, error: {error}" - ) + async def retrieve() -> Any: + return _status_result(await result_getter(), wait_statuses, success_status) - return result + return await self._wait_until_complete_async(retrieve) def _wait_for_extraction( self, diff --git a/packages/uipath-platform/src/uipath/platform/documents/_documents_service_v2.py b/packages/uipath-platform/src/uipath/platform/documents/_documents_service_v2.py index b4924d19e..10ce1666b 100644 --- a/packages/uipath-platform/src/uipath/platform/documents/_documents_service_v2.py +++ b/packages/uipath-platform/src/uipath/platform/documents/_documents_service_v2.py @@ -1,6 +1,47 @@ -from typing import Any, Dict, Tuple +import asyncio +from contextlib import nullcontext +from pathlib import Path +from typing import Any, Dict, Optional, Tuple +from uuid import UUID -from ._documents_service import _DocumentsServiceBase # type: ignore[attr-defined] +from uipath.core.tracing import traced + +from ..common._config import UiPathApiConfig +from ..common._execution_context import UiPathExecutionContext +from ..common._folder_context import header_folder +from ..common._models import Endpoint +from ..orchestrator._folder_service import FolderService +from ._documents_service import ( # type: ignore[attr-defined] + POLLING_INTERVAL, + POLLING_TIMEOUT, + _DocumentsServiceBase, + _exactly_one_must_be_provided, + _operation_result, + _url_path, +) +from .documents import ( + FileContent, + ModelExtractionResponse, + StartModelExtractionResponse, +) + + +def _models_url(*segments: Any) -> Endpoint: + return Endpoint(f"/du_/api/framework/models/{_url_path(segments)}") + + +def _parse_folder_key(folder_key: Optional[str]) -> str: + if not folder_key: + raise ValueError( + "A folder is required for a model deployed to a folder: pass `folder_key` " + "or `folder_path`, or run in a job folder (`UIPATH_FOLDER_KEY` or " + "`UIPATH_FOLDER_PATH`)." + ) + try: + # The folder routes accept only a GUID folder key. + return str(UUID(folder_key)) + except ValueError: + raise ValueError(f"`folder_key` must be a GUID: {folder_key!r}") from None class DocumentsServiceV2(_DocumentsServiceBase): @@ -12,6 +53,9 @@ class DocumentsServiceV2(_DocumentsServiceBase): - IXP results group fields in `FieldGroup` fields instead of `Table` fields, and their fields are typed, e.g. `MonetaryQuantity` - validation tasks open in the Compact view + It can also extract with IXP models deployed to an Orchestrator folder, which API v1.1 can't reach: + `extract_with_model`, or `start_model_extraction` and `retrieve_model_extraction_result`. + !!! warning "Preview Feature" This function is currently experimental. Behavior and parameters are subject to change in future versions. @@ -19,6 +63,22 @@ class DocumentsServiceV2(_DocumentsServiceBase): _api_version = "2.0" + def __init__( + self, + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + polling_interval: float = POLLING_INTERVAL, + polling_timeout: float = POLLING_TIMEOUT, + folders_service: Optional[FolderService] = None, + ) -> None: + super().__init__( + config=config, + execution_context=execution_context, + polling_interval=polling_interval, + polling_timeout=polling_timeout, + ) + self._folders_service = folders_service + def _tag_route(self, tag: str) -> Tuple[str, ...]: # v2 moved tag-scoped routes under a literal `tags` segment. return ("tags", tag) @@ -26,3 +86,361 @@ def _tag_route(self, tag: str) -> Tuple[str, ...]: def _allow_document_type_change(self) -> Dict[str, Any]: # v2 moved this setting into the validation configuration. return {"configuration": {"allowChangeOfDocumentType": True}} + + def _folders(self) -> FolderService: + if self._folders_service is None: + raise ValueError( + "Looking up a folder path's key needs a `FolderService`: use " + "`UiPath().documents_v2`, or pass `folders_service` or `folder_key`." + ) + return self._folders_service + + def _folder_to_resolve( + self, folder_key: Optional[str], folder_path: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + """Returns the folder key, or else the folder path to look its key up by. + + Explicit arguments come first, then the job folder from the environment. + """ + if folder_key is not None and folder_path is not None: + raise ValueError( + "Only one of `folder_key` or `folder_path` can be provided" + ) + if folder_key is not None or folder_path is not None: + return folder_key, folder_path + if self._folder_key: + return self._folder_key, None + return None, self._folder_path or None + + def _model_folder_key( + self, folder_key: Optional[str], folder_path: Optional[str] + ) -> str: + folder_key, folder_path = self._folder_to_resolve(folder_key, folder_path) + if folder_path: + folder_key = self._folders().retrieve_folder_key(folder_path) + return _parse_folder_key(folder_key) + + async def _model_folder_key_async( + self, folder_key: Optional[str], folder_path: Optional[str] + ) -> str: + folder_key, folder_path = self._folder_to_resolve(folder_key, folder_path) + if folder_path: + folder_key = await self._folders().retrieve_folder_key_async(folder_path) + return _parse_folder_key(folder_key) + + def _model_headers(self, folder_key: str) -> Dict[str, str]: + return { + **self._get_common_headers(), + **header_folder(_parse_folder_key(folder_key), None), + } + + def _wait_for_model_digitization( + self, model_name: str, folder_key: str, document_id: str + ) -> None: + def result_getter() -> Tuple[str, Optional[str], Optional[str]]: + result = self._request_json( + "GET", + url=_models_url(model_name, "digitization", "result", document_id), + params={"api-version": self._api_version}, + headers=self._model_headers(folder_key), + ) + return result["status"], result.get("error"), None + + self._wait_for_operation( + result_getter=result_getter, + wait_statuses=["NotStarted", "Running"], + success_status="Succeeded", + ) + + async def _wait_for_model_digitization_async( + self, model_name: str, folder_key: str, document_id: str + ) -> None: + async def result_getter() -> Tuple[str, Optional[str], Optional[str]]: + result = await self._request_json_async( + "GET", + url=_models_url(model_name, "digitization", "result", document_id), + params={"api-version": self._api_version}, + headers=self._model_headers(folder_key), + ) + return result["status"], result.get("error"), None + + await self._wait_for_operation_async( + result_getter=result_getter, + wait_statuses=["NotStarted", "Running"], + success_status="Succeeded", + ) + + @traced(name="documents_start_model_extraction", run_type="uipath") + def start_model_extraction( + self, + model_name: str, + file: Optional[FileContent] = None, + file_path: Optional[str] = None, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> StartModelExtractionResponse: + """Start an extraction with an IXP model deployed to a folder, without waiting for the result (non-blocking). + + It digitizes the document, waiting for the digitization to finish, then starts the extraction. + + Args: + model_name (str): Name of the model deployed to the folder, e.g. `invoices-ixp`. This is the model's URL name, not its display name. + file (FileContent, optional): The document file to be processed. + file_path (str, optional): Path to the document file to be processed. + folder_key (str, optional): Key of the folder the model is deployed to. + folder_path (str, optional): Path of the folder the model is deployed to. Its key is looked up. + + Note: + Either `file` or `file_path` must be provided, but not both. + Pass at most one of `folder_key` and `folder_path`. Without either, the current job's folder is used. + + Returns: + StartModelExtractionResponse: Contains the operation_id, document_id, model_name, and folder_key. + + Examples: + ```python + start_response = uipath.documents_v2.start_model_extraction( + model_name="invoices-ixp", + file_path="path/to/document.pdf", + folder_path="Shared/Invoices", + ) + # Later, read the result: + extraction_response = uipath.documents_v2.retrieve_model_extraction_result(start_response) + ``` + """ + _exactly_one_must_be_provided(file=file, file_path=file_path) + _url_path((model_name,)) + folder_key = self._model_folder_key(folder_key, folder_path) + headers = self._model_headers(folder_key) + + with open(Path(file_path), "rb") if file_path else nullcontext(file) as handle: + document_id = self._request_json( + "POST", + url=_models_url(model_name, "digitization", "start"), + params={"api-version": self._api_version}, + headers=headers, + files={"File": handle}, + )["documentId"] + + self._wait_for_model_digitization(model_name, folder_key, document_id) + + operation_id = self._request_json( + "POST", + url=_models_url(model_name, "extraction", "start"), + params={"api-version": self._api_version}, + headers=headers, + json={"documentId": document_id}, + )["operationId"] + + return StartModelExtractionResponse( + operation_id=operation_id, + document_id=document_id, + model_name=model_name, + folder_key=folder_key, + ) + + @traced(name="documents_start_model_extraction_async", run_type="uipath") + async def start_model_extraction_async( + self, + model_name: str, + file: Optional[FileContent] = None, + file_path: Optional[str] = None, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> StartModelExtractionResponse: + """Asynchronous version of the [`start_model_extraction`][uipath.platform.documents._documents_service_v2.DocumentsServiceV2.start_model_extraction] method.""" + _exactly_one_must_be_provided(file=file, file_path=file_path) + _url_path((model_name,)) + folder_key = await self._model_folder_key_async(folder_key, folder_path) + headers = self._model_headers(folder_key) + + # Read in a worker thread so the event loop isn't blocked. Keeping the + # file name keeps the upload's content type, which DU checks. + upload = ( + (Path(file_path).name, await asyncio.to_thread(Path(file_path).read_bytes)) + if file_path + else file + ) + document_id = ( + await self._request_json_async( + "POST", + url=_models_url(model_name, "digitization", "start"), + params={"api-version": self._api_version}, + headers=headers, + files={"File": upload}, + ) + )["documentId"] + + await self._wait_for_model_digitization_async( + model_name, folder_key, document_id + ) + + operation_id = ( + await self._request_json_async( + "POST", + url=_models_url(model_name, "extraction", "start"), + params={"api-version": self._api_version}, + headers=headers, + json={"documentId": document_id}, + ) + )["operationId"] + + return StartModelExtractionResponse( + operation_id=operation_id, + document_id=document_id, + model_name=model_name, + folder_key=folder_key, + ) + + def _model_extraction_result_url( + self, start_response: StartModelExtractionResponse + ) -> Endpoint: + return _models_url( + start_response.model_name, + "extraction", + "result", + start_response.operation_id, + ) + + def _model_extraction_response( + self, response: Dict[str, Any], start_response: StartModelExtractionResponse + ) -> ModelExtractionResponse: + result = _operation_result( + response, start_response.operation_id, "Model extraction" + ) + return ModelExtractionResponse.model_validate( + { + **result, + "modelName": start_response.model_name, + "folderKey": start_response.folder_key, + } + ) + + def _retrieve_model_extraction_result( + self, start_response: StartModelExtractionResponse + ) -> ModelExtractionResponse: + response = self._request_json( + "GET", + url=self._model_extraction_result_url(start_response), + params={"api-version": self._api_version}, + headers=self._model_headers(start_response.folder_key), + ) + return self._model_extraction_response(response, start_response) + + async def _retrieve_model_extraction_result_async( + self, start_response: StartModelExtractionResponse + ) -> ModelExtractionResponse: + response = await self._request_json_async( + "GET", + url=self._model_extraction_result_url(start_response), + params={"api-version": self._api_version}, + headers=self._model_headers(start_response.folder_key), + ) + return self._model_extraction_response(response, start_response) + + @traced(name="documents_retrieve_model_extraction_result", run_type="uipath") + def retrieve_model_extraction_result( + self, start_response: StartModelExtractionResponse + ) -> ModelExtractionResponse: + """Retrieve the result of an extraction started with `start_model_extraction` (single-shot, non-blocking). + + It makes a single request and returns the result if available, or raises an exception if not complete. + + Args: + start_response (StartModelExtractionResponse): The response returned by `start_model_extraction`. + + Returns: + ModelExtractionResponse: The extraction response containing the extracted data. + + Raises: + OperationNotCompleteException: If the extraction is not yet complete. + OperationFailedException: If the extraction operation failed. + + Examples: + ```python + extraction_response = uipath.documents_v2.retrieve_model_extraction_result(start_response) + ``` + """ + return self._retrieve_model_extraction_result(start_response) + + @traced(name="documents_retrieve_model_extraction_result_async", run_type="uipath") + async def retrieve_model_extraction_result_async( + self, start_response: StartModelExtractionResponse + ) -> ModelExtractionResponse: + """Asynchronous version of the [`retrieve_model_extraction_result`][uipath.platform.documents._documents_service_v2.DocumentsServiceV2.retrieve_model_extraction_result] method.""" + return await self._retrieve_model_extraction_result_async(start_response) + + @traced(name="documents_extract_with_model", run_type="uipath") + def extract_with_model( + self, + model_name: str, + file: Optional[FileContent] = None, + file_path: Optional[str] = None, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> ModelExtractionResponse: + """Extract data from a document with an IXP model deployed to a folder, waiting for the result. + + It is `start_model_extraction` followed by `retrieve_model_extraction_result` until the result is ready. + + Args: + model_name (str): Name of the model deployed to the folder, e.g. `invoices-ixp`. This is the model's URL name, not its display name. + file (FileContent, optional): The document file to be processed. + file_path (str, optional): Path to the document file to be processed. + folder_key (str, optional): Key of the folder the model is deployed to. + folder_path (str, optional): Path of the folder the model is deployed to. Its key is looked up. + + Note: + Either `file` or `file_path` must be provided, but not both. + Pass at most one of `folder_key` and `folder_path`. Without either, the current job's folder is used. + + Returns: + ModelExtractionResponse: The extraction response containing the extracted data. + + Raises: + OperationFailedException: If the extraction operation failed. + TimeoutError: If the extraction didn't finish within the polling timeout. + + Examples: + ```python + with open("path/to/document.pdf", "rb") as file: + extraction_response = uipath.documents_v2.extract_with_model( + model_name="invoices-ixp", + file=file, + folder_path="Shared/Invoices", + ) + ``` + """ + start_response = self.start_model_extraction( + model_name=model_name, + file=file, + file_path=file_path, + folder_key=folder_key, + folder_path=folder_path, + ) + + return self._wait_until_complete( + lambda: self._retrieve_model_extraction_result(start_response) + ) + + @traced(name="documents_extract_with_model_async", run_type="uipath") + async def extract_with_model_async( + self, + model_name: str, + file: Optional[FileContent] = None, + file_path: Optional[str] = None, + folder_key: Optional[str] = None, + folder_path: Optional[str] = None, + ) -> ModelExtractionResponse: + """Asynchronous version of the [`extract_with_model`][uipath.platform.documents._documents_service_v2.DocumentsServiceV2.extract_with_model] method.""" + start_response = await self.start_model_extraction_async( + model_name=model_name, + file=file, + file_path=file_path, + folder_key=folder_key, + folder_path=folder_path, + ) + + return await self._wait_until_complete_async( + lambda: self._retrieve_model_extraction_result_async(start_response) + ) diff --git a/packages/uipath-platform/src/uipath/platform/documents/documents.py b/packages/uipath-platform/src/uipath/platform/documents/documents.py index f21f998d5..0a8c34fe1 100644 --- a/packages/uipath-platform/src/uipath/platform/documents/documents.py +++ b/packages/uipath-platform/src/uipath/platform/documents/documents.py @@ -280,7 +280,20 @@ class ClassificationResponse(BaseModel): ) -class StartOperationResponse(BaseModel): +class _StartOperationBase(BaseModel): + """The fields that every started operation has, whatever addresses its model.""" + + model_config = ConfigDict( + serialize_by_alias=True, + validate_by_alias=True, + validate_by_name=True, + ) + + operation_id: str = Field(alias="operationId") + document_id: str = Field(alias="documentId") + + +class StartOperationResponse(_StartOperationBase): """A model representing the response from starting an operation. Attributes: @@ -291,14 +304,6 @@ class StartOperationResponse(BaseModel): api_version (str): The DU Framework API version that started the operation, `1.1` or `2.0`. """ - model_config = ConfigDict( - serialize_by_alias=True, - validate_by_alias=True, - validate_by_name=True, - ) - - operation_id: str = Field(alias="operationId") - document_id: str = Field(alias="documentId") project_id: str = Field(alias="projectId") tag: str | None = Field(default=None) api_version: DocumentsApiVersion = Field(alias="apiVersion", default="1.1") @@ -308,5 +313,45 @@ class StartExtractionResponse(StartOperationResponse): """A model representing the response from starting an extraction operation.""" +class StartModelExtractionResponse(_StartOperationBase): + """A model representing the response from starting an extraction with a model deployed to a folder. + + Pass it to `DocumentsServiceV2.retrieve_model_extraction_result` to read the result. + It isn't a `StartOperationResponse`: a model deployed to a folder has no project or tag. + + Attributes: + operation_id (str): The ID of the extraction operation, used to poll for results. + document_id (str): The ID of the digitized document. + model_name (str): The name of the model deployed to the folder. + folder_key (str): The key of the folder the model is deployed to. + """ + + model_name: str = Field(alias="modelName") + folder_key: str = Field(alias="folderKey") + + +class ModelExtractionResponse(BaseModel): + """A model representing the result of an extraction with a model deployed to a folder. + + Unlike project extraction results, it has no data projection: DU Framework + doesn't return one for models deployed to a folder. + + Attributes: + extraction_result (ExtractionResult): The result of the extraction process. + model_name (str): The name of the model deployed to the folder. + folder_key (str): The key of the folder the model is deployed to. + """ + + model_config = ConfigDict( + serialize_by_alias=True, + validate_by_alias=True, + validate_by_name=True, + ) + + extraction_result: ExtractionResult = Field(alias="extractionResult") + model_name: str = Field(alias="modelName") + folder_key: str = Field(alias="folderKey") + + class StartExtractionValidationResponse(StartOperationResponse): """A model representing the response from starting an extraction validation operation.""" diff --git a/packages/uipath-platform/tests/services/test_documents_service.py b/packages/uipath-platform/tests/services/test_documents_service.py index a63760456..3ab4388df 100644 --- a/packages/uipath-platform/tests/services/test_documents_service.py +++ b/packages/uipath-platform/tests/services/test_documents_service.py @@ -1,7 +1,7 @@ import json from pathlib import Path from typing import Any -from unittest.mock import Mock, patch +from unittest.mock import AsyncMock, Mock, patch from uuid import UUID, uuid4 import pytest @@ -3109,6 +3109,87 @@ async def mock_result_getter_async(): success_status="Succeeded", ) + @pytest.mark.parametrize("mode", ["sync", "async"]) + @pytest.mark.asyncio + @patch("uipath.platform.documents._documents_service.asyncio") + @patch("uipath.platform.documents._documents_service.time") + async def test_wait_for_operation_sleeps_only_between_checks( + self, + mock_time: Mock, + mock_asyncio: Mock, + service: DocumentsService, + mode: str, + ): + # ARRANGE + mock_time.monotonic.return_value = 0 + mock_asyncio.sleep = AsyncMock() + outcomes = [ + ("NotStarted", None, None), + ("Running", None, None), + ("Succeeded", None, {"done": True}), + ] + + def mock_result_getter(): + return outcomes.pop(0) + + async def mock_result_getter_async(): + return outcomes.pop(0) + + # ACT + if mode == "async": + result = await service._wait_for_operation_async( + result_getter=mock_result_getter_async, + wait_statuses=["NotStarted", "Running"], + success_status="Succeeded", + ) + else: + result = service._wait_for_operation( + result_getter=mock_result_getter, + wait_statuses=["NotStarted", "Running"], + success_status="Succeeded", + ) + + # ASSERT + assert result == {"done": True} + sleep = mock_asyncio.sleep if mode == "async" else mock_time.sleep + # Between the three checks, and not after the last one. + assert sleep.call_count == 2 + + @pytest.mark.parametrize("mode", ["sync", "async"]) + @pytest.mark.asyncio + @patch("uipath.platform.documents._documents_service.time") + async def test_wait_for_operation_reports_a_failure_near_the_deadline( + self, + mock_time: Mock, + service: DocumentsService, + mode: str, + ): + # ARRANGE + # The failure comes back from a check that ends after the timeout. + mock_time.monotonic.side_effect = [0, 5, 12] + outcomes = [("Running", None, None), ("Failed", "Dummy error", None)] + + def mock_result_getter(): + return outcomes.pop(0) + + async def mock_result_getter_async(): + return outcomes.pop(0) + + # ACT & ASSERT + with pytest.raises(RuntimeError, match="status: Failed, error: Dummy error"): + if mode == "async": + await service._wait_for_operation_async( + result_getter=mock_result_getter_async, + wait_statuses=["NotStarted", "Running"], + success_status="Succeeded", + ) + else: + service._wait_for_operation( + result_getter=mock_result_getter, + wait_statuses=["NotStarted", "Running"], + success_status="Succeeded", + ) + @pytest.mark.parametrize("mode", ["sync", "async"]) @pytest.mark.asyncio async def test_start_ixp_extraction( diff --git a/packages/uipath-platform/tests/services/test_documents_service_v2.py b/packages/uipath-platform/tests/services/test_documents_service_v2.py index 5706dc572..2f667c343 100644 --- a/packages/uipath-platform/tests/services/test_documents_service_v2.py +++ b/packages/uipath-platform/tests/services/test_documents_service_v2.py @@ -1,18 +1,25 @@ import json from pathlib import Path -from typing import Any +from typing import Any, Callable from uuid import UUID, uuid4 import pytest from pytest_httpx import HTTPXMock -from uipath.platform import UiPathApiConfig, UiPathExecutionContext +from uipath.platform import UiPath, UiPathApiConfig, UiPathExecutionContext from uipath.platform.documents import ( DocumentsServiceV2, FieldType, + ModelExtractionResponse, ProjectType, + StartModelExtractionResponse, ValidateExtractionAction, ) +from uipath.platform.errors import ( + OperationFailedException, + OperationNotCompleteException, +) +from uipath.platform.orchestrator import FolderService # Responses only DU Framework API v2 returns. Behavior both versions share is # tested against both services in test_documents_service.py. @@ -255,3 +262,411 @@ async def test_get_validate_extraction_result_pascal_case_responses( "Default.Details.Total" ) assert response.tag == "live" + + +FOLDER_KEY = str(uuid4()) + + +@pytest.fixture +def make_service( + config: UiPathApiConfig, + execution_context: UiPathExecutionContext, + monkeypatch: pytest.MonkeyPatch, +) -> Callable[..., DocumentsServiceV2]: + """Builds a service after the test sets the job folder environment.""" + monkeypatch.delenv("UIPATH_FOLDER_KEY", raising=False) + monkeypatch.delenv("UIPATH_FOLDER_PATH", raising=False) + + def make( + polling_timeout: float = 10, with_folders_service: bool = True + ) -> DocumentsServiceV2: + return DocumentsServiceV2( + config=config, + execution_context=execution_context, + polling_interval=0.001, + polling_timeout=polling_timeout, + folders_service=( + FolderService(config=config, execution_context=execution_context) + if with_folders_service + else None + ), + ) + + return make + + +@pytest.fixture +def model_extraction_result(ixp_extraction_response_v2: dict) -> dict: # type: ignore + return {"extractionResult": ixp_extraction_response_v2["extractionResult"]} + + +async def call( + mode: str, service: DocumentsServiceV2, method: str, **kwargs: Any +) -> Any: + """Calls `method`, or its `_async` twin in async mode.""" + if mode == "async": + return await getattr(service, f"{method}_async")(**kwargs) + return getattr(service, method)(**kwargs) + + +class TestDocumentsServiceV2FolderModels: + @pytest.fixture + def models_url(self, base_url: str, org: str, tenant: str) -> str: + return f"{base_url}{org}{tenant}/du_/api/framework/models" + + def mock_start( + self, + httpx_mock: HTTPXMock, + models_url: str, + model_route: str, + folder_key: str, + document_id: str, + operation_id: str, + pascal_case: bool = False, + ) -> None: + def body(value: dict) -> dict: # type: ignore + return to_pascal_case(value) if pascal_case else value + + headers = { + "X-UiPath-Internal-ConsumptionSourceType": "CodedAgents", + "x-uipath-folderkey": folder_key, + } + httpx_mock.add_response( + method="POST", + url=f"{models_url}/{model_route}/digitization/start?api-version=2.0", + match_headers=headers, + json=body({"documentId": document_id}), + ) + for status in ["NotStarted", "Running", "Succeeded"]: + httpx_mock.add_response( + method="GET", + url=f"{models_url}/{model_route}/digitization/result/{document_id}?api-version=2.0", + match_headers=headers, + json=body({"status": status}), + ) + httpx_mock.add_response( + method="POST", + url=f"{models_url}/{model_route}/extraction/start?api-version=2.0", + match_headers=headers, + match_json={"documentId": document_id}, + json=body({"operationId": operation_id}), + ) + + @pytest.mark.parametrize("mode", ["sync", "async"]) + @pytest.mark.parametrize("pascal_case", [False, True]) + @pytest.mark.asyncio + async def test_extract_with_model( + self, + httpx_mock: HTTPXMock, + make_service: Callable[..., DocumentsServiceV2], + models_url: str, + mode: str, + pascal_case: bool, + model_extraction_result: dict, # type: ignore + ): + # ARRANGE + service = make_service() + document_id, operation_id = str(uuid4()), str(uuid4()) + self.mock_start( + httpx_mock, + models_url, + "my%20model", + FOLDER_KEY, + document_id, + operation_id, + pascal_case, + ) + for status in ["NotStarted", "Running", "Succeeded"]: + body = {"status": status, "result": model_extraction_result} + httpx_mock.add_response( + method="GET", + url=f"{models_url}/my%20model/extraction/result/{operation_id}?api-version=2.0", + match_headers={"x-uipath-folderkey": FOLDER_KEY}, + json=to_pascal_case(body) if pascal_case else body, + ) + + # ACT + if mode == "async": + response = await service.extract_with_model_async( + model_name="my model", file=b"test content", folder_key=FOLDER_KEY + ) + else: + response = service.extract_with_model( + model_name="my model", file=b"test content", folder_key=FOLDER_KEY + ) + + # ASSERT + assert isinstance(response, ModelExtractionResponse) + assert response.model_name == "my model" + assert response.folder_key == FOLDER_KEY + assert ( + response.extraction_result.model_dump() + == model_extraction_result["extractionResult"] + ) + [upload] = [ + request + for request in httpx_mock.get_requests() + if request.url.path.endswith("/digitization/start") + ] + assert b"test content" in upload.read() + + @pytest.mark.parametrize("mode", ["sync", "async"]) + @pytest.mark.asyncio + async def test_start_and_retrieve_model_extraction( + self, + httpx_mock: HTTPXMock, + make_service: Callable[..., DocumentsServiceV2], + models_url: str, + mode: str, + model_extraction_result: dict, # type: ignore + tmp_path: Path, + ): + # ARRANGE + service = make_service() + document = tmp_path / "invoice.pdf" + document.write_bytes(b"test content") + document_id, operation_id = str(uuid4()), str(uuid4()) + self.mock_start( + httpx_mock, models_url, "invoices", FOLDER_KEY, document_id, operation_id + ) + result_url = ( + f"{models_url}/invoices/extraction/result/{operation_id}?api-version=2.0" + ) + httpx_mock.add_response(url=result_url, json={"status": "Running"}) + httpx_mock.add_response( + url=result_url, + json={"status": "Succeeded", "result": model_extraction_result}, + ) + + # ACT + if mode == "async": + start_response = await service.start_model_extraction_async( + model_name="invoices", file_path=str(document), folder_key=FOLDER_KEY + ) + else: + start_response = service.start_model_extraction( + model_name="invoices", file_path=str(document), folder_key=FOLDER_KEY + ) + # Resume triggers keep the start response as a dict and rebuild it. + restored = StartModelExtractionResponse.model_validate( + json.loads(start_response.model_dump_json()) + ) + + # ASSERT + [upload] = [ + request + for request in httpx_mock.get_requests() + if request.url.path.endswith("/digitization/start") + ] + # DU checks the content type, which comes from the file name. + body = upload.read() + assert b'filename="invoice.pdf"' in body + assert b"Content-Type: application/pdf" in body + assert b"test content" in body + assert restored == StartModelExtractionResponse( + operation_id=operation_id, + document_id=document_id, + model_name="invoices", + folder_key=FOLDER_KEY, + ) + with pytest.raises(OperationNotCompleteException): + await call( + mode, + service, + "retrieve_model_extraction_result", + start_response=restored, + ) + + if mode == "async": + response = await service.retrieve_model_extraction_result_async(restored) + else: + response = service.retrieve_model_extraction_result(restored) + assert ( + response.extraction_result.document_id + == (model_extraction_result["extractionResult"]["DocumentId"]) + ) + + @pytest.mark.parametrize("mode", ["sync", "async"]) + @pytest.mark.asyncio + async def test_retrieve_model_extraction_result_failed( + self, + httpx_mock: HTTPXMock, + make_service: Callable[..., DocumentsServiceV2], + models_url: str, + mode: str, + ): + # ARRANGE + service = make_service() + start_response = StartModelExtractionResponse( + operation_id="op", document_id="doc", model_name="m", folder_key=FOLDER_KEY + ) + httpx_mock.add_response( + url=f"{models_url}/m/extraction/result/op?api-version=2.0", + json={"status": "Failed", "error": "Model not found"}, + ) + + # ACT & ASSERT + with pytest.raises(OperationFailedException, match="Model not found"): + await call( + mode, + service, + "retrieve_model_extraction_result", + start_response=start_response, + ) + + @pytest.mark.parametrize("mode", ["sync", "async"]) + @pytest.mark.asyncio + async def test_extract_with_model_times_out( + self, + httpx_mock: HTTPXMock, + make_service: Callable[..., DocumentsServiceV2], + models_url: str, + mode: str, + ): + # ARRANGE + service = make_service(polling_timeout=0.05) + document_id, operation_id = str(uuid4()), str(uuid4()) + self.mock_start( + httpx_mock, models_url, "invoices", FOLDER_KEY, document_id, operation_id + ) + httpx_mock.add_response( + url=f"{models_url}/invoices/extraction/result/{operation_id}?api-version=2.0", + json={"status": "Running"}, + is_reusable=True, + ) + + # ACT & ASSERT + with pytest.raises(TimeoutError): + await call( + mode, + service, + "extract_with_model", + model_name="invoices", + file=b"x", + folder_key=FOLDER_KEY, + ) + + @pytest.mark.parametrize("mode", ["sync", "async"]) + @pytest.mark.parametrize( + "env, folder_path", + [ + # The job folder from the environment. + ({"UIPATH_FOLDER_KEY": FOLDER_KEY}, None), + # A job folder path, looked up. + ({"UIPATH_FOLDER_PATH": "Shared/Invoices"}, None), + # An explicit folder path wins over the job folder. + ({"UIPATH_FOLDER_KEY": str(uuid4())}, "Shared/Invoices"), + ], + ids=["env-key", "env-path", "explicit-path"], + ) + @pytest.mark.asyncio + async def test_start_model_extraction_resolves_folder( + self, + httpx_mock: HTTPXMock, + make_service: Callable[..., DocumentsServiceV2], + monkeypatch: pytest.MonkeyPatch, + base_url: str, + org: str, + tenant: str, + models_url: str, + mode: str, + env: dict, # type: ignore + folder_path: str, + ): + # ARRANGE + for name, value in env.items(): + monkeypatch.setenv(name, value) + service = make_service() + if folder_path or "UIPATH_FOLDER_PATH" in env: + httpx_mock.add_response( + url=f"{base_url}{org}{tenant}/orchestrator_/api/FoldersNavigation/GetFoldersForCurrentUser?searchText=Invoices&skip=0&take=20", + json={ + "PageItems": [ + {"Key": FOLDER_KEY, "FullyQualifiedName": "Shared/Invoices"} + ] + }, + ) + document_id, operation_id = str(uuid4()), str(uuid4()) + self.mock_start( + httpx_mock, models_url, "invoices", FOLDER_KEY, document_id, operation_id + ) + + # ACT + if mode == "async": + start_response = await service.start_model_extraction_async( + model_name="invoices", file=b"x", folder_path=folder_path + ) + else: + start_response = service.start_model_extraction( + model_name="invoices", file=b"x", folder_path=folder_path + ) + + # ASSERT + assert start_response.folder_key == FOLDER_KEY + + @pytest.mark.parametrize("mode", ["sync", "async"]) + @pytest.mark.parametrize( + "kwargs, message", + [ + ({}, "A folder is required"), + ( + {"folder_key": FOLDER_KEY, "folder_path": "Shared"}, + "Only one of `folder_key` or `folder_path`", + ), + ({"folder_key": "not-a-guid"}, "`folder_key` must be a GUID"), + ({"model_name": "", "folder_key": FOLDER_KEY}, "Invalid URL path segment"), + ({"model_name": ".", "folder_key": FOLDER_KEY}, "Invalid URL path segment"), + ( + {"model_name": "..", "folder_key": FOLDER_KEY}, + "Invalid URL path segment", + ), + ], + ids=["no-folder", "key-and-path", "bad-key", "empty-name", "dot", "dot-dot"], + ) + @pytest.mark.asyncio + async def test_start_model_extraction_rejects_invalid_arguments( + self, + make_service: Callable[..., DocumentsServiceV2], + mode: str, + kwargs: dict, # type: ignore + message: str, + ): + # ARRANGE + service = make_service() + kwargs = {"model_name": "invoices", "file": b"x", **kwargs} + + # ACT & ASSERT: it fails before sending any request. + with pytest.raises(ValueError, match=message): + await call(mode, service, "start_model_extraction", **kwargs) + + @pytest.mark.parametrize("mode", ["sync", "async"]) + @pytest.mark.asyncio + async def test_folder_path_needs_a_folders_service( + self, + make_service: Callable[..., DocumentsServiceV2], + mode: str, + ): + # ARRANGE + service = make_service(with_folders_service=False) + + # ACT & ASSERT: it fails before sending any request. + with pytest.raises(ValueError, match="needs a `FolderService`"): + await call( + mode, + service, + "start_model_extraction", + model_name="invoices", + file=b"x", + folder_path="Shared/Invoices", + ) + + +def test_only_documents_v2_extracts_with_folder_models( + config: UiPathApiConfig, +) -> None: + sdk = UiPath(base_url=config.base_url, secret=config.secret) + + assert hasattr(sdk.documents_v2, "extract_with_model") + assert isinstance(sdk.documents_v2._folders(), FolderService) + assert not hasattr(sdk.documents, "extract_with_model") + assert not hasattr(sdk.documents, "start_model_extraction") diff --git a/packages/uipath-platform/uv.lock b/packages/uipath-platform/uv.lock index 479b3bdca..17d0ca82d 100644 --- a/packages/uipath-platform/uv.lock +++ b/packages/uipath-platform/uv.lock @@ -1082,7 +1082,7 @@ dev = [ [[package]] name = "uipath-platform" -version = "0.3.12" +version = "0.3.13" source = { editable = "." } dependencies = [ { name = "anyio" }, diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index ab608bcd9..ffe400b67 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -2883,7 +2883,7 @@ wheels = [ [[package]] name = "uipath-platform" -version = "0.3.12" +version = "0.3.13" source = { editable = "../uipath-platform" } dependencies = [ { name = "anyio" },