fix(temporal): deduplicate tracing and skip replay logs

adk/README.md

@@ -27,6 +27,26 @@ This automatically pulls in [`agentex-client`](../) (the slim Stainless-generate
 
 The two packages contribute disjoint files to the `agentex.*` namespace — `agentex/lib/*` ships only from `agentex-sdk`.
 
+## Workflow logging
+
+Use the workflow logger in Temporal workflow code:
+
+```python
+from agentex.lib.core.temporal.logging import make_workflow_logger
+
+logger = make_workflow_logger(__name__)
+```
+
+It suppresses logs while Temporal replays recorded history and adds top-level
+`workflow_id` and `run_id` fields during workflow execution. It preserves the
+message, caller fields, and exception details. Outside workflows, including in
+activities, it behaves like the ordinary SDK logger.
+
+New Temporal templates use this helper. Existing agents must replace their own
+workflow loggers to get the same behavior. This does not create trace context or
+add trace IDs to workflows that lack it. Temporal's worker diagnostics still report
+replay failures.
+
 ## Repo layout
 
 This package is hand-authored and lives at `adk/` inside [scaleapi/scale-agentex-python](https://github.com/scaleapi/scale-agentex-python). Stainless codegen never touches `adk/**` — it's outside the generated surface. The sibling `agentex-client` package lives at the repo root and IS Stainless-generated.

src/agentex/lib/adk/utils/_modules/client.py

@@ -1,3 +1,4 @@
+import os
 from typing import override
 
 import httpx
@@ -26,7 +27,50 @@ def auth_flow(self, request):
         yield request
 
 
+# HTTP timeouts for the AgentEx client, in seconds. Defaults match the SDK's
+# DEFAULT_TIMEOUT, so leaving these unset changes nothing.
+_TIMEOUT_ENV_DEFAULTS = {
+    "connect": ("AGENTEX_CLIENT_CONNECT_TIMEOUT_SECONDS", 5.0),
+    "read": ("AGENTEX_CLIENT_READ_TIMEOUT_SECONDS", 300.0),
+    "write": ("AGENTEX_CLIENT_WRITE_TIMEOUT_SECONDS", 300.0),
+    "pool": ("AGENTEX_CLIENT_POOL_TIMEOUT_SECONDS", 300.0),
+}
+
+
+def _timeout_from_env() -> httpx.Timeout:
+    """Build the client timeout from environment variables.
+
+    Read from ``os.environ`` rather than from ``EnvironmentVariables``. That model
+    is loaded by worker startup and by ``EnvAuth.auth_flow`` on every request, and
+    ``agentex.lib.adk.utils`` builds a client at import time, so a field added
+    there would make a malformed timeout break all three. Reading here keeps the
+    blast radius to the one value that is actually wrong.
+
+    The connect timeout is the one worth raising: an AgentEx backend accepts
+    connections serially, so connect latency grows with the number of callers and
+    the 5s default is reached when a few hundred are in flight.
+    """
+    values = {}
+    for field, (env_var, default) in _TIMEOUT_ENV_DEFAULTS.items():
+        raw = os.environ.get(env_var)
+        if raw is None or raw.strip() == "":
+            values[field] = default
+            continue
+        try:
+            values[field] = float(raw)
+        except ValueError as exc:
+            raise ValueError(f"{env_var} must be a number in seconds, got {raw!r}") from exc
+    return httpx.Timeout(**values)
+
+
 def create_async_agentex_client(**kwargs) -> AsyncAgentex:
+    """Create an AsyncAgentex client.
+
+    An explicit ``timeout=`` always wins; otherwise the timeout comes from the
+    AGENTEX_CLIENT_*_TIMEOUT_SECONDS environment variables.
+    """
+    if "timeout" not in kwargs:
+        kwargs["timeout"] = _timeout_from_env()
     client = AsyncAgentex(**kwargs)
     client._client.auth = EnvAuth()
     return client

src/agentex/lib/cli/handlers/deploy_handlers.py

@@ -389,6 +389,8 @@ def merge_deployment_configs(
             _deep_merge(helm_values, agent_env_config.helm_overrides)
         logger.info(f"After-merge helm values: {helm_values}")
 
+    _stamp_agent_version(helm_values, set(all_env_vars) | {var["name"] for var in secret_env_vars})
+
     # Set final environment variables
     # Environment variable precedence: manifest -> environments.yaml -> secrets (highest)
     if all_env_vars:
@@ -430,6 +432,14 @@ def _deep_merge(base_dict: dict[str, Any], override_dict: dict[str, Any]) -> Non
             base_dict[key] = value
 
 
+def _stamp_agent_version(helm_values: dict[str, Any], declared_env_names: set[str]) -> None:
+    """Set global.agent.version from the merged image tag unless the deployment declares AGENT_VERSION itself."""
+    if EnvVarKeys.AGENT_VERSION.value in declared_env_names:
+        # Chart >=0.6.0 renders global.agent.version as a second AGENT_VERSION env entry.
+        return
+    helm_values["global"]["agent"].setdefault("version", helm_values["global"]["image"]["tag"])
+
+
 def create_helm_values_file(helm_values: dict[str, Any]) -> str:
     """Create a temporary helm values file"""
     with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f:

src/agentex/lib/cli/templates/temporal-claude-code/project/workflow.py.j2

@@ -27,7 +27,7 @@ from temporalio import workflow
 from agentex.lib import adk
 from agentex.lib.types.acp import SendEventParams, CreateTaskParams
 from agentex.lib.types.tracing import SGPTracingProcessorConfig
-from agentex.lib.utils.logging import make_logger
+from agentex.lib.core.temporal.logging import make_workflow_logger
 from agentex.types.text_content import TextContent
 from agentex.lib.environment_variables import EnvironmentVariables
 from agentex.lib.core.temporal.types.workflow import SignalName
@@ -52,7 +52,7 @@ if environment_variables.WORKFLOW_NAME is None:
 if environment_variables.AGENT_NAME is None:
     raise ValueError("Environment variable AGENT_NAME is not set")
 
-logger = make_logger(__name__)
+logger = make_workflow_logger(__name__)
 
 
 @workflow.defn(name=environment_variables.WORKFLOW_NAME)

src/agentex/lib/cli/templates/temporal-codex/project/workflow.py.j2

@@ -29,7 +29,7 @@ from temporalio import workflow
 from agentex.lib import adk
 from agentex.lib.types.acp import SendEventParams, CreateTaskParams
 from agentex.lib.types.tracing import SGPTracingProcessorConfig
-from agentex.lib.utils.logging import make_logger
+from agentex.lib.core.temporal.logging import make_workflow_logger
 from agentex.types.text_content import TextContent
 from agentex.lib.environment_variables import EnvironmentVariables
 from agentex.lib.core.temporal.types.workflow import SignalName
@@ -54,7 +54,7 @@ if environment_variables.WORKFLOW_NAME is None:
 if environment_variables.AGENT_NAME is None:
     raise ValueError("Environment variable AGENT_NAME is not set")
 
-logger = make_logger(__name__)
+logger = make_workflow_logger(__name__)
 
 MODEL = os.environ.get("CODEX_MODEL", "o4-mini")
 

src/agentex/lib/cli/templates/temporal-langgraph/project/workflow.py.j2

@@ -37,7 +37,7 @@ from project.graph import GRAPH_NAME, build_graph
 from agentex.lib.adk import emit_langgraph_messages
 from agentex.protocol.acp import SendEventParams, CreateTaskParams
 from agentex.lib.types.tracing import SGPTracingProcessorConfig
-from agentex.lib.utils.logging import make_logger
+from agentex.lib.core.temporal.logging import make_workflow_logger
 from agentex.types.text_content import TextContent
 from agentex.lib.environment_variables import EnvironmentVariables
 from agentex.lib.core.temporal.types.workflow import SignalName
@@ -64,7 +64,7 @@ if environment_variables.WORKFLOW_NAME is None:
 if environment_variables.AGENT_NAME is None:
     raise ValueError("Environment variable AGENT_NAME is not set")
 
-logger = make_logger(__name__)
+logger = make_workflow_logger(__name__)
 
 
 @workflow.defn(name=environment_variables.WORKFLOW_NAME)

src/agentex/lib/cli/templates/temporal-openai-agents/project/workflow.py.j2

@@ -7,7 +7,7 @@ from agentex.lib import adk
 from agentex.protocol.acp import CreateTaskParams, SendEventParams
 from agentex.lib.core.temporal.workflows.workflow import BaseWorkflow
 from agentex.lib.core.temporal.types.workflow import SignalName
-from agentex.lib.utils.logging import make_logger
+from agentex.lib.core.temporal.logging import make_workflow_logger
 from agentex.types.text_content import TextContent
 from agentex.lib.environment_variables import EnvironmentVariables
 from agents import Agent, Runner, set_tracing_disabled
@@ -37,7 +37,7 @@ if environment_variables.WORKFLOW_NAME is None:
 if environment_variables.AGENT_NAME is None:
     raise ValueError("Environment variable AGENT_NAME is not set")
 
-logger = make_logger(__name__)
+logger = make_workflow_logger(__name__)
 
 # Setup tracing for SGP (Scale GenAI Platform)
 # This enables visibility into your agent's execution in the SGP dashboard

src/agentex/lib/cli/templates/temporal-pydantic-ai/project/workflow.py.j2

@@ -25,7 +25,7 @@ from project.agent import TaskDeps, temporal_agent
 from agentex.lib import adk
 from agentex.protocol.acp import SendEventParams, CreateTaskParams
 from agentex.lib.types.tracing import SGPTracingProcessorConfig
-from agentex.lib.utils.logging import make_logger
+from agentex.lib.core.temporal.logging import make_workflow_logger
 from agentex.types.text_content import TextContent
 from agentex.lib.environment_variables import EnvironmentVariables
 from agentex.lib.core.temporal.types.workflow import SignalName
@@ -55,7 +55,7 @@ if environment_variables.WORKFLOW_NAME is None:
 if environment_variables.AGENT_NAME is None:
     raise ValueError("Environment variable AGENT_NAME is not set")
 
-logger = make_logger(__name__)
+logger = make_workflow_logger(__name__)
 
 
 @workflow.defn(name=environment_variables.WORKFLOW_NAME)

src/agentex/lib/cli/templates/temporal/project/workflow.py.j2

@@ -6,7 +6,7 @@ from agentex.lib import adk
 from agentex.protocol.acp import CreateTaskParams, SendEventParams
 from agentex.lib.core.temporal.workflows.workflow import BaseWorkflow
 from agentex.lib.core.temporal.types.workflow import SignalName
-from agentex.lib.utils.logging import make_logger
+from agentex.lib.core.temporal.logging import make_workflow_logger
 from agentex.types.text_content import TextContent
 from agentex.lib.environment_variables import EnvironmentVariables
 
@@ -18,7 +18,7 @@ if environment_variables.WORKFLOW_NAME is None:
 if environment_variables.AGENT_NAME is None:
     raise ValueError("Environment variable AGENT_NAME is not set")
 
-logger = make_logger(__name__)
+logger = make_workflow_logger(__name__)
 
 @workflow.defn(name=environment_variables.WORKFLOW_NAME)
 class {{ workflow_class }}(BaseWorkflow):

src/agentex/lib/core/temporal/logging.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import Any, override
+from collections.abc import MutableMapping
+
+from temporalio import workflow
+
+from agentex.lib.utils.logging import make_logger
+
+
+class WorkflowLoggerAdapter(workflow.LoggerAdapter):
+    """Skip workflow replay logs and add IDs without changing non-workflow logs."""
+
+    @override
+    def isEnabledFor(self, level: int) -> bool:
+        if not workflow.in_workflow():
+            return self.logger.isEnabledFor(level)
+        return super().isEnabledFor(level)
+
+    @override
+    def process(self, msg: Any, kwargs: MutableMapping[str, Any]) -> tuple[Any, MutableMapping[str, Any]]:
+        if workflow.in_workflow():
+            info = workflow.info()
+            kwargs["extra"] = {
+                "workflow_id": info.workflow_id,
+                "run_id": info.run_id,
+                **(kwargs.get("extra") or {}),
+            }
+        return msg, kwargs
+
+
+def make_workflow_logger(name: str) -> WorkflowLoggerAdapter:
+    """Create an SDK logger that suppresses replay and adds workflow/run IDs."""
+    return WorkflowLoggerAdapter(make_logger(name), {})

src/agentex/lib/core/temporal/plugins/openai_agents/interceptors/context_interceptor.py

@@ -22,8 +22,10 @@
 )
 from temporalio.converter import default
 
+from agentex.lib.core.temporal.logging import WorkflowLoggerAdapter
+
 # Set up logging
-logger = logging.getLogger("context.interceptor")
+logger = WorkflowLoggerAdapter(logging.getLogger("context.interceptor"), {})
 
 # Global context variables that models can read
 # These are thread-safe and work across async boundaries

src/agentex/lib/core/temporal/workers/worker.py

@@ -258,9 +258,8 @@ async def run(
             max_concurrent_activities=self.max_concurrent_activities,
             build_id=str(uuid.uuid4()),
             debug_mode=debug_enabled,  # Disable deadlock detection in debug mode
-            # Tracing interceptor OUTERMOST so business interceptors (and the spans
-            # they create) nest under the propagated workflow/activity span.
-            interceptors=[*temporal_tracing_interceptors(), *self.interceptors],
+            # Temporal inherits client tracing before these business interceptors.
+            interceptors=self.interceptors,
         )
 
         logger.info(f"Starting workers for task queue: {self.task_queue}")

src/agentex/lib/core/temporal/workflows/workflow.py

@@ -7,10 +7,10 @@
 from temporalio import workflow
 
 from agentex.protocol.acp import SendEventParams, CreateTaskParams, InterruptTaskParams
-from agentex.lib.utils.logging import make_logger
+from agentex.lib.core.temporal.logging import make_workflow_logger
 from agentex.lib.core.temporal.types.workflow import SignalName
 
-logger = make_logger(__name__)
+logger = make_workflow_logger(__name__)
 
 
 class BaseWorkflow(ABC):

src/agentex/lib/core/tracing/code_revision.py

@@ -1,10 +1,11 @@
-"""Opt-in stamping of the agent's source commit onto its spans.
+"""Stamping of the agent's source commit onto its spans.
 
-Nothing is stamped until the agent calls :func:`enable`, mirroring the
-``lineage`` registry next door: a process-wide switch the agent sets once at
-import, rather than automatic behaviour every agent inherits. When enabled the
-resolved commit lands in span data under ``__commit_sha__`` and is searchable in
-the SGP Traces UI as ``__commit_sha__:<sha>``.
+Stamping turns on when the process starts with ``AGENT_COMMIT_SHA`` set, which
+the SGP cloud deploy does from the build record's attested commit, or when the
+agent calls :func:`enable` itself. Nothing is stamped otherwise: upgrading the
+SDK alone never starts emitting the field. When on, the resolved commit lands in
+span data under ``__commit_sha__`` and is searchable in the SGP Traces UI as
+``__commit_sha__:<sha>``.
 
 This is deliberately separate from ``__agent_version__``, which is automatic and
 carries the deployed image tag verbatim ("image tag or git sha"). That tag is a
@@ -21,7 +22,7 @@
 
 from agentex.lib.utils.logging import make_logger
 
-__all__ = ("COMMIT_SHA_KEY", "enable", "disable", "is_enabled", "commit_sha")
+__all__ = ("COMMIT_SHA_KEY", "enable", "disable", "is_enabled", "commit_sha", "is_git_object_name")
 
 logger = make_logger(__name__)
 
@@ -31,6 +32,12 @@
 # git's own 7-character minimum.
 _GIT_SHA_RE = re.compile(r"[0-9a-fA-F]{7,64}")
 
+
+def is_git_object_name(value: str) -> bool:
+    """Whether ``value`` is a full or abbreviated git SHA-1/SHA-256 object name."""
+    return _GIT_SHA_RE.fullmatch(value.strip()) is not None
+
+
 _COMMIT_SHA_ENV = "AGENT_COMMIT_SHA"
 # Fallback only: automatic, and only usable when it happens to be SHA-shaped.
 _AGENT_VERSION_ENV = "AGENT_VERSION"
@@ -42,13 +49,16 @@
 
 
 def enable(commit_sha: str | None = None) -> None:
-    """Opt this process in to stamping ``__commit_sha__`` onto every span.
+    """Turn on stamping ``__commit_sha__`` onto every span from this process.
 
     Value precedence: the explicit ``commit_sha`` argument, else
     ``AGENT_COMMIT_SHA``, else ``AGENT_VERSION`` when the deployment happened to
     set it to a bare commit SHA. A value that is not a git object name is
     refused with a warning and leaves stamping off -- better an absent field
     than one named for a commit that holds an image tag.
+
+    Called once at import when ``AGENT_COMMIT_SHA`` is set, so a deployment that
+    supplies the commit needs no code change in the agent.
     """
     global _commit_sha
 
@@ -103,3 +113,12 @@ def is_enabled() -> bool:
 def commit_sha() -> str | None:
     """The resolved commit SHA, or ``None`` when stamping is not enabled."""
     return _commit_sha
+
+
+def _enable_from_environment() -> None:
+    """Auto-enable on ``AGENT_COMMIT_SHA`` only; ``AGENT_VERSION`` stays an explicit fallback."""
+    if os.environ.get(_COMMIT_SHA_ENV, "").strip():
+        enable()
+
+
+_enable_from_environment()

src/agentex/lib/environment_variables.py

@@ -26,6 +26,7 @@ class EnvVarKeys(str, Enum):
     AGENT_ID = "AGENT_ID"
     AGENT_VERSION = "AGENT_VERSION"
     AGENT_COMMIT_SHA = "AGENT_COMMIT_SHA"
+    AGENT_SOURCE_REPO = "AGENT_SOURCE_REPO"
     AGENT_API_KEY = "AGENT_API_KEY"
     # ACP Configuration
     ACP_URL = "ACP_URL"
@@ -68,12 +69,11 @@ class EnvironmentVariables(BaseModel):
     AGENT_ID: str | None = None
     # Build/version discriminator (image tag or git sha), set by the deployment
     AGENT_VERSION: str | None = None
-    # The agent's source commit, baked into the image or set by the deployment.
-    # Unlike AGENT_VERSION this is expected to be a git SHA and nothing else, and
-    # it is OPT-IN: nothing is stamped unless the agent calls
-    # `adk.code_revision.enable()`, which also refuses a value that is not a git
-    # object name. See agentex.lib.core.tracing.code_revision.
+    # The agent's source commit, set by the deployment or baked into the image; a git
+    # SHA and nothing else. Stamped as __commit_sha__ when set (see tracing.code_revision).
     AGENT_COMMIT_SHA: str | None = None
+    # Git remote the agent was built from (any URL form; normalized to host/path on use).
+    AGENT_SOURCE_REPO: str | None = None
     AGENT_API_KEY: str | None = None
     ACP_TYPE: str | None = "async"
     AGENT_INPUT_TYPE: str | None = None

src/agentex/lib/utils/build_provenance.py

@@ -82,7 +82,8 @@ def normalize_remote(url: Optional[str]) -> Optional[str]:
     """Strip credentials and scheme from a remote, returning ``host/path``."""
     if not url:
         return None
-    candidate = url.strip()
+    # Query strings and fragments never name a repo, but they do carry tokens.
+    candidate = url.strip().split("?", 1)[0].split("#", 1)[0]
     # scp-like syntax: git@host:org/repo(.git) — no scheme, host/path split on ':'
     if "://" not in candidate and ":" in candidate and "/" not in candidate.split(":", 1)[0]:
         candidate = candidate.split("@", 1)[-1].replace(":", "/", 1)