feat: add configurable HTTP retry strategy with exponential backoff

databusclient/api/delete.py

@@ -14,6 +14,7 @@
 from databusclient.api.utils import (
     fetch_databus_jsonld,
     get_databus_id_parts_from_file_url,
+    get_http_session,
 )
 
 
@@ -23,16 +24,26 @@ class DeleteQueue:
     Allows adding multiple databus URIs to a queue and executing their deletion in batch.
     """
 
-    def __init__(self, databus_key: str, manifest_context=None):
+    def __init__(
+        self,
+        databus_key: str,
+        manifest_context=None,
+        session: requests.Session | None = None,
+        timeout: int = 30,
+    ):
         """Create a DeleteQueue bound to a given Databus API key.
 
         Args:
             databus_key: API key used to authenticate deletion requests.
             manifest_context: Optional ManifestContext to record deletion
                 outcomes into. Passed through to _delete_list on execute().
+            session: Optional HTTP session with retry strategy.
+            timeout: Request timeout in seconds.
         """
         self.databus_key = databus_key
         self.manifest_context = manifest_context
+        self.session = session
+        self.timeout = timeout
         self.queue: set[str] = set()
 
     def add_uri(self, databusURI: str):
@@ -79,6 +90,8 @@ def execute(self):
             self.databus_key,
             force=True,
             manifest_context=self.manifest_context,
+            session=self.session,
+            timeout=self.timeout,
         )
 
 
@@ -122,6 +135,8 @@ def _delete_resource(
     force: bool = False,
     queue: DeleteQueue = None,
     manifest_context=None,
+    session: requests.Session | None = None,
+    timeout: int = 30,
 ):
     """Delete a single Databus resource (version, artifact, group).
 
@@ -134,6 +149,8 @@ def _delete_resource(
         dry_run: If True, do not perform the deletion but only print what would be deleted.
         force: If True, skip confirmation prompt and proceed with deletion.
         queue: If queue is provided, add the URI to the queue instead of deleting immediately.
+        session: Optional HTTP session to use for requests.
+        timeout: Request timeout in seconds.
     """
 
     # Confirm the deletion request, skip the request or cancel deletion process
@@ -158,9 +175,12 @@ def _delete_resource(
         queue.add_uri(databusURI)
         return
 
+    if session is None:
+        session = get_http_session()
+
     print(f"[DELETE] {databusURI}")
     headers = {"accept": "*/*", "X-API-KEY": databus_key}
-    response = requests.delete(databusURI, headers=headers, timeout=30)
+    response = session.delete(databusURI, headers=headers, timeout=timeout)
 
     if response.status_code in (200, 204):
         print(f"Successfully deleted: {databusURI}")
@@ -179,6 +199,8 @@ def _delete_list(
     force: bool = False,
     queue: DeleteQueue = None,
     manifest_context=None,
+    session: requests.Session | None = None,
+    timeout: int = 30,
 ):
     """Delete a list of Databus resources.
 
@@ -188,10 +210,19 @@ def _delete_list(
         dry_run: If True, do not perform the deletion but only print what would be deleted.
         force: If True, skip confirmation prompt and proceed with deletion.
         queue: If queue is provided, add the URIs to the queue instead of deleting immediately.
+        session: Optional HTTP session with retry strategy.
+        timeout: Request timeout in seconds.
     """
     for databusURI in databusURIs:
         _delete_resource(
-            databusURI, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
+            databusURI,
+            databus_key,
+            dry_run=dry_run,
+            force=force,
+            queue=queue,
+            manifest_context=manifest_context,
+            session=session,
+            timeout=timeout,
         )
 
 
@@ -202,6 +233,8 @@ def _delete_artifact(
     force: bool = False,
     queue: DeleteQueue = None,
     manifest_context=None,
+    session: requests.Session | None = None,
+    timeout: int = 30,
 ):
     """Delete an artifact and all its versions.
 
@@ -214,8 +247,10 @@ def _delete_artifact(
         dry_run: If True, do not perform the deletion but only print what would be deleted.
         force: If True, skip confirmation prompt and proceed with deletion.
         queue: If queue is provided, add the URI to the queue instead of deleting immediately.
+        session: Optional HTTP session with retry strategy.
+        timeout: Request timeout in seconds.
     """
-    artifact_body = fetch_databus_jsonld(databusURI, databus_key)
+    artifact_body = fetch_databus_jsonld(databusURI, databus_key, session=session, timeout=timeout)
 
     json_dict = json.loads(artifact_body)
     versions = json_dict.get("databus:hasVersion")
@@ -235,11 +270,27 @@ def _delete_artifact(
         else:
             # Delete all versions
             _delete_list(
-                version_uris, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
+                version_uris,
+                databus_key,
+                dry_run=dry_run,
+                force=force,
+                queue=queue,
+                manifest_context=manifest_context,
+                session=session,
+                timeout=timeout,
             )
 
     # Finally, delete the artifact itself
-    _delete_resource(databusURI, databus_key, dry_run=dry_run, force=force, queue=queue,manifest_context=manifest_context)
+    _delete_resource(
+        databusURI,
+        databus_key,
+        dry_run=dry_run,
+        force=force,
+        queue=queue,
+        manifest_context=manifest_context,
+        session=session,
+        timeout=timeout,
+    )
 
 
 def _delete_group(
@@ -249,6 +300,8 @@ def _delete_group(
     force: bool = False,
     queue: DeleteQueue = None,
     manifest_context=None,
+    session: requests.Session | None = None,
+    timeout: int = 30,
 ):
     """Delete a group and all its artifacts and versions.
 
@@ -261,8 +314,10 @@ def _delete_group(
         dry_run: If True, do not perform the deletion but only print what would be deleted.
         force: If True, skip confirmation prompt and proceed with deletion.
         queue: If queue is provided, add the URI to the queue instead of deleting immediately.
+        session: Optional HTTP session with retry strategy.
+        timeout: Request timeout in seconds.
     """
-    group_body = fetch_databus_jsonld(databusURI, databus_key)
+    group_body = fetch_databus_jsonld(databusURI, databus_key, session=session, timeout=timeout)
 
     json_dict = json.loads(group_body)
     artifacts = json_dict.get("databus:hasArtifact", [])
@@ -279,14 +334,38 @@ def _delete_group(
     # Delete all artifacts (which deletes their versions)
     for artifact_uri in artifact_uris:
         _delete_artifact(
-            artifact_uri, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
+            artifact_uri,
+            databus_key,
+            dry_run=dry_run,
+            force=force,
+            queue=queue,
+            manifest_context=manifest_context,
+            session=session,
+            timeout=timeout,
         )
 
     # Finally, delete the group itself
-    _delete_resource(databusURI, databus_key, dry_run=dry_run, force=force, queue=queue,manifest_context=manifest_context)
+    _delete_resource(
+        databusURI,
+        databus_key,
+        dry_run=dry_run,
+        force=force,
+        queue=queue,
+        manifest_context=manifest_context,
+        session=session,
+        timeout=timeout,
+    )
 
 
-def delete(databusURIs: List[str], databus_key: str, dry_run: bool, force: bool, manifest_context=None):
+def delete(
+    databusURIs: List[str],
+    databus_key: str,
+    dry_run: bool = False,
+    force: bool = False,
+    manifest_context=None,
+    session: requests.Session | None = None,
+    timeout: int = 30,
+):
     """Delete a dataset from the databus.
 
     Delete a group, artifact, or version identified by the given databus URI.
@@ -297,9 +376,15 @@ def delete(databusURIs: List[str], databus_key: str, dry_run: bool, force: bool,
         databus_key: Databus API key to authenticate the deletion requests.
         dry_run: If True, will only print what would be deleted without performing actual deletions.
         force: If True, skip confirmation prompt and proceed with deletion.
+        session: Optional HTTP session with retry strategy.
+        timeout: Request timeout in seconds.
     """
+    if session is None:
+        session = get_http_session()
 
-    queue = DeleteQueue(databus_key, manifest_context=manifest_context)
+    queue = DeleteQueue(
+        databus_key, manifest_context=manifest_context, session=session, timeout=timeout
+    )
 
     for databusURI in databusURIs:
         _host, _account, group, artifact, version, file = (
@@ -309,24 +394,52 @@ def delete(databusURIs: List[str], databus_key: str, dry_run: bool, force: bool,
         if group == "collections" and artifact is not None:
             print(f"Deleting collection: {databusURI}")
             _delete_resource(
-                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
+                databusURI,
+                databus_key,
+                dry_run=dry_run,
+                force=force,
+                queue=queue,
+                manifest_context=manifest_context,
+                session=session,
+                timeout=timeout,
             )
         elif file is not None:
             print(f"Deleting file is not supported via API: {databusURI}")
         elif version is not None:
             print(f"Deleting version: {databusURI}")
             _delete_resource(
-                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
+                databusURI,
+                databus_key,
+                dry_run=dry_run,
+                force=force,
+                queue=queue,
+                manifest_context=manifest_context,
+                session=session,
+                timeout=timeout,
             )
         elif artifact is not None:
             print(f"Deleting artifact and all its versions: {databusURI}")
             _delete_artifact(
-                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
+                databusURI,
+                databus_key,
+                dry_run=dry_run,
+                force=force,
+                queue=queue,
+                manifest_context=manifest_context,
+                session=session,
+                timeout=timeout,
             )
         elif group is not None and group != "collections":
             print(f"Deleting group and all its artifacts and versions: {databusURI}")
             _delete_group(
-                databusURI, databus_key, dry_run=dry_run, force=force, queue=queue, manifest_context=manifest_context
+                databusURI,
+                databus_key,
+                dry_run=dry_run,
+                force=force,
+                queue=queue,
+                manifest_context=manifest_context,
+                session=session,
+                timeout=timeout,
             )
         else:
             print(f"Deleting {databusURI} is not supported.")

databusclient/api/deploy.py

@@ -12,6 +12,8 @@
 
 import requests
 
+from databusclient.api.utils import get_http_session
+
 _debug = False
 
 
@@ -176,14 +178,18 @@ def _get_file_stats(distribution_str: str) -> Tuple[Optional[str], Optional[int]
     return sha256sum, content_length
 
 
-def _load_file_stats(url: str) -> Tuple[str, int]:
+def _load_file_stats(
+    url: str, session: requests.Session | None = None, timeout: int = 30
+) -> Tuple[str, int]:
     """Download the file at ``url`` and compute its SHA-256 and length.
 
     This is used as a fallback when the caller did not supply checksum/size
     information in the CLI or metadata file.
     """
+    if session is None:
+        session = get_http_session()
 
-    resp = requests.get(url, timeout=30)
+    resp = session.get(url, timeout=timeout)
     if resp.status_code >= 400:
         raise requests.exceptions.RequestException(response=resp)
 
@@ -463,8 +469,11 @@ def deploy(
     verify_parts: bool = False,
     log_level: DeployLogLevel = DeployLogLevel.debug,
     debug: bool = False,
+    session: requests.Session | None = None,
+    timeout: int = 30,
 ) -> None:
-    """Deploys a dataset to the databus. The endpoint is inferred from the DataID identifier.
+    """Deploy a Databus Dataset (JSON-LD structure) to the Databus.
+
     Parameters
     ----------
     dataid: Dict[str, Union[List[Dict[str, Union[bool, str, int, float, List]]], str]]
@@ -477,7 +486,13 @@ def deploy(
         log level of the deploy output
     debug: bool
         controls whether output shold be printed to the console (stdout)
+    session: requests.Session
+        optional HTTP session with retry strategy
+    timeout: int
+        HTTP request timeout in seconds
     """
+    if session is None:
+        session = get_http_session()
 
     headers = {"X-API-KEY": f"{api_key}", "Content-Type": "application/json"}
     data = json.dumps(dataid)
@@ -491,7 +506,7 @@ def deploy(
         base
         + f"/api/publish?verify-parts={str(verify_parts).lower()}&log-level={log_level.name}"
     )
-    resp = requests.post(api_uri, data=data, headers=headers, timeout=30)
+    resp = session.post(api_uri, data=data, headers=headers, timeout=timeout)
 
     if debug or _debug:
         try:
@@ -517,6 +532,8 @@ def deploy_from_metadata(
     artifact_version_description: str,
     license_url: str,
     apikey: str,
+    session: requests.Session | None = None,
+    timeout: int = 30,
 ) -> None:
     """
     Deploy a dataset from metadata entries.
@@ -537,6 +554,10 @@ def deploy_from_metadata(
         License URI
     apikey : str
         API key for authentication
+    session : requests.Session
+        Optional HTTP session with retry strategy
+    timeout : int
+        HTTP request timeout in seconds
     """
     distributions = create_distributions_from_metadata(metadata)
 
@@ -550,7 +571,7 @@ def deploy_from_metadata(
     )
 
     print(f"Deploying dataset version: {version_id}")
-    deploy(dataset, apikey)
+    deploy(dataset, apikey, session=session, timeout=timeout)
 
     print(f"Successfully deployed to {version_id}")
     print(f"Deployed {len(metadata)} file(s):")