feat(obs): wire sgp-obs from the SDK for traces, metrics and logs

adk/pyproject.toml

@@ -65,6 +65,7 @@ dependencies = [
 # agentex/lib/* uses `from typing import override` (3.12+) in 19 files.
 # The slim agentex-client keeps 3.11 support.
 requires-python = ">= 3.12,<4"
+
 classifiers = [
   "Typing :: Typed",
   "Intended Audience :: Developers",
@@ -76,6 +77,23 @@ classifiers = [
   "License :: OSI Approved :: Apache Software License",
 ]
 
+# No `obs` extra, deliberately — do not add one for sgp-obs.
+#
+# sgp-obs is not on public PyPI (it is served from Scale's curated CodeArtifact
+# mirror), and declaring it in [project.optional-dependencies] makes THIS repo's uv
+# workspace unresolvable: `uv sync` re-locks, locking must resolve every declared
+# optional dependency of every workspace member, and there is no way to exempt one.
+# Measured: `uv lock --check`, `uv sync --all-extras`, plain `uv sync` with no extras,
+# and `uv sync --all-extras --no-extra obs` all fail (`--no-extra` filters what is
+# installed, not what is resolved); `uv lock` has no `--no-extra`; and
+# `[tool.uv] override-dependencies` does not exempt it either. Only `--frozen` works,
+# which would leave nobody able to re-lock this repo again.
+#
+# So the dependency is the AGENT's to declare — `sgp-obs[genai-auto,http,otlp]`
+# against the mirror — and the SDK wires it when it is importable. See
+# agentex/lib/core/observability/sgp_obs_setup.py; nothing imports sgp_obs outside a
+# try, so a plain `pip install agentex-sdk` is unaffected either way.
+
 [project.urls]
 Homepage = "https://github.com/scaleapi/scale-agentex-python"
 Repository = "https://github.com/scaleapi/scale-agentex-python"

src/agentex/lib/cli/templates/PRIVATE_INDEX.md

@@ -0,0 +1,62 @@
+# The private package index in scaffold Dockerfiles
+
+Every scaffold Dockerfile mounts a build secret named `codeartifact-pip-conf`. It lets an agent
+install Scale-internal packages — `sgp-obs`, for instance — that are not on public PyPI, without the
+build holding any registry credential of its own. The control-plane broker mints a short-lived
+CodeArtifact token per build and injects it as that secret.
+
+- Design: [Private Package Access for Customer Agents (PRD)](https://app.notion.com/p/Private-Package-Access-for-Customer-Agents-PRD-3ad904d6e6cb802cb091df1c25e230bc)
+- Tracking: [SGPINF-1568](https://linear.app/scale-epd/issue/SGPINF-1568/provide-scale-internal-packages-to-agentex-agents-in-customer)
+
+## It is inert by default
+
+The mount is `required=false` and guarded by `[ -s ... ]`, so with no secret injected the build is
+byte-identical to one without any of this. That covers every local build, every CI build, and every
+agent that never opts in. An empty secret file is skipped too.
+
+## Opting in
+
+Add the index to the agent's `pyproject.toml`:
+
+```toml
+[[tool.uv.index]]
+name = "scale-pypi"
+url = "<the scale-customer-pypi URL>"
+default = true
+```
+
+The name must be exactly `scale-pypi`. uv applies `UV_INDEX_SCALE_PYPI_USERNAME` /
+`UV_INDEX_SCALE_PYPI_PASSWORD` to the index of that name, so renaming it makes the credentials
+silently stop applying. Setting `UV_INDEX_URL` instead does not authenticate a *named* index at
+all, and the resolve fails with a 401.
+
+## Three things that are easy to get wrong
+
+**The token arrives percent-encoded.** The buildspec URL-encodes it to embed it in the pip config's
+URL userinfo, so a token containing `+`, `/` or `=` arrives as `%2B`, `%2F`, `%3D`. The `uv sync`
+templates decode it before exporting it as a password. Passing it through still-encoded sends a
+different string and the resolve 401s.
+
+**The credential must not follow project-controlled configuration.** uv binds credentials by index
+*name*, and the name-to-URL mapping would otherwise come from the agent's own `pyproject.toml` — so a
+project that pointed `scale-pypi` at another host would receive the token. Verified against a local
+server: the rogue host receives `Authorization: Basic aws:<token>` and the real index is never
+contacted. The templates therefore export `UV_INDEX` to re-bind the name to the URL the *broker*
+supplied, which overrides whatever the project declared. With that in place the rogue host is never
+contacted. The pinned URL carries no userinfo; the token still travels only in
+`UV_INDEX_SCALE_PYPI_PASSWORD`.
+
+The case this defends is not a malicious agent author — they also write the Dockerfile and could read
+the mounted secret directly. It is a *contributed* change to a project file, where a one-line URL edit
+is far less conspicuous in review than an exfiltration command in a Dockerfile.
+
+**The two template variants work differently, deliberately.**
+
+| Template | Install step | How the credential is supplied |
+| --- | --- | --- |
+| `Dockerfile-uv.j2` | `uv sync` against the agent's `pyproject.toml` | Named index `scale-pypi`, pinned via `UV_INDEX`, token decoded into `UV_INDEX_SCALE_PYPI_PASSWORD` |
+| `Dockerfile.j2` | `uv pip install -r requirements.txt` | No pyproject is present, so there is no named index to bind to. The credentialed URL is used directly via `UV_DEFAULT_INDEX` |
+
+The `requirements.txt` variant does **not** decode the token, and that is the point: it stays inside
+the URL, already encoded for exactly that use. Decoding it there would corrupt it. It is also not
+exposed to the redirection problem above, because the URL comes wholly from the injected secret.

src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2

@@ -34,14 +34,34 @@ WORKDIR /app/{{ project_path_from_build_root }}
 COPY {{ project_path_from_build_root }}/pyproject.toml ./
 
 # Install dependencies (without project itself, for layer caching)
+# Optional private index for Scale-internal packages such as sgp-obs, injected by the
+# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local
+# builds, CI builds, and agents that never opt in are unaffected.
+#
+# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting
+# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory.
 RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=secret,id=codeartifact-pip-conf,required=false \
+    if [ -s /run/secrets/codeartifact-pip-conf ]; then \
+      export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \
+      export UV_INDEX_SCALE_PYPI_USERNAME=aws; \
+      export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \
+        | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \
+    fi; \
     uv sync --no-install-project --no-dev
 
 # Copy the project code
 COPY {{ project_path_from_build_root }}/project ./project
 
 # Install the project
 RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=secret,id=codeartifact-pip-conf,required=false \
+    if [ -s /run/secrets/codeartifact-pip-conf ]; then \
+      export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \
+      export UV_INDEX_SCALE_PYPI_USERNAME=aws; \
+      export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \
+        | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \
+    fi; \
     uv sync --no-dev
 
 ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH"

src/agentex/lib/cli/templates/default-claude-code/Dockerfile.j2

@@ -33,8 +33,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr
 
 WORKDIR /app/{{ project_path_from_build_root }}
 
+# Optional private index for Scale-internal packages such as sgp-obs, injected by the
+# control-plane broker (SGPINF-1568). Inert unless the secret is present.
+#
+# This variant installs from requirements.txt, so there is no pyproject.toml for uv to
+# read a named index out of; the credentialed URL is used directly and is deliberately
+# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory.
+#
 # Install the required Python packages
-RUN uv pip install --system -r requirements.txt
+RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \
+    if [ -s /run/secrets/codeartifact-pip-conf ]; then \
+      export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \
+    fi; \
+    uv pip install --system -r requirements.txt
 
 # Copy the project code
 COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project

src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2

@@ -34,14 +34,34 @@ WORKDIR /app/{{ project_path_from_build_root }}
 COPY {{ project_path_from_build_root }}/pyproject.toml ./
 
 # Install dependencies (without project itself, for layer caching)
+# Optional private index for Scale-internal packages such as sgp-obs, injected by the
+# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local
+# builds, CI builds, and agents that never opt in are unaffected.
+#
+# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting
+# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory.
 RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=secret,id=codeartifact-pip-conf,required=false \
+    if [ -s /run/secrets/codeartifact-pip-conf ]; then \
+      export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \
+      export UV_INDEX_SCALE_PYPI_USERNAME=aws; \
+      export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \
+        | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \
+    fi; \
     uv sync --no-install-project --no-dev
 
 # Copy the project code
 COPY {{ project_path_from_build_root }}/project ./project
 
 # Install the project
 RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=secret,id=codeartifact-pip-conf,required=false \
+    if [ -s /run/secrets/codeartifact-pip-conf ]; then \
+      export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \
+      export UV_INDEX_SCALE_PYPI_USERNAME=aws; \
+      export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \
+        | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \
+    fi; \
     uv sync --no-dev
 
 ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH"

src/agentex/lib/cli/templates/default-codex/Dockerfile.j2

@@ -33,8 +33,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr
 
 WORKDIR /app/{{ project_path_from_build_root }}
 
+# Optional private index for Scale-internal packages such as sgp-obs, injected by the
+# control-plane broker (SGPINF-1568). Inert unless the secret is present.
+#
+# This variant installs from requirements.txt, so there is no pyproject.toml for uv to
+# read a named index out of; the credentialed URL is used directly and is deliberately
+# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory.
+#
 # Install the required Python packages
-RUN uv pip install --system -r requirements.txt
+RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \
+    if [ -s /run/secrets/codeartifact-pip-conf ]; then \
+      export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \
+    fi; \
+    uv pip install --system -r requirements.txt
 
 # Copy the project code
 COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project

src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2

@@ -30,14 +30,34 @@ WORKDIR /app/{{ project_path_from_build_root }}
 COPY {{ project_path_from_build_root }}/pyproject.toml ./
 
 # Install dependencies (without project itself, for layer caching)
+# Optional private index for Scale-internal packages such as sgp-obs, injected by the
+# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local
+# builds, CI builds, and agents that never opt in are unaffected.
+#
+# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting
+# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory.
 RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=secret,id=codeartifact-pip-conf,required=false \
+    if [ -s /run/secrets/codeartifact-pip-conf ]; then \
+      export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \
+      export UV_INDEX_SCALE_PYPI_USERNAME=aws; \
+      export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \
+        | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \
+    fi; \
     uv sync --no-install-project --no-dev
 
 # Copy the project code
 COPY {{ project_path_from_build_root }}/project ./project
 
 # Install the project
 RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=secret,id=codeartifact-pip-conf,required=false \
+    if [ -s /run/secrets/codeartifact-pip-conf ]; then \
+      export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \
+      export UV_INDEX_SCALE_PYPI_USERNAME=aws; \
+      export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \
+        | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \
+    fi; \
     uv sync --no-dev
 
 ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH"

src/agentex/lib/cli/templates/default-langgraph/Dockerfile.j2

@@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr
 
 WORKDIR /app/{{ project_path_from_build_root }}
 
+# Optional private index for Scale-internal packages such as sgp-obs, injected by the
+# control-plane broker (SGPINF-1568). Inert unless the secret is present.
+#
+# This variant installs from requirements.txt, so there is no pyproject.toml for uv to
+# read a named index out of; the credentialed URL is used directly and is deliberately
+# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory.
+#
 # Install the required Python packages
-RUN uv pip install --system -r requirements.txt
+RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \
+    if [ -s /run/secrets/codeartifact-pip-conf ]; then \
+      export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \
+    fi; \
+    uv pip install --system -r requirements.txt
 
 # Copy the project code
 COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project

src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2

@@ -30,14 +30,34 @@ WORKDIR /app/{{ project_path_from_build_root }}
 COPY {{ project_path_from_build_root }}/pyproject.toml ./
 
 # Install dependencies (without project itself, for layer caching)
+# Optional private index for Scale-internal packages such as sgp-obs, injected by the
+# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local
+# builds, CI builds, and agents that never opt in are unaffected.
+#
+# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting
+# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory.
 RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=secret,id=codeartifact-pip-conf,required=false \
+    if [ -s /run/secrets/codeartifact-pip-conf ]; then \
+      export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \
+      export UV_INDEX_SCALE_PYPI_USERNAME=aws; \
+      export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \
+        | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \
+    fi; \
     uv sync --no-install-project --no-dev
 
 # Copy the project code
 COPY {{ project_path_from_build_root }}/project ./project
 
 # Install the project
 RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=secret,id=codeartifact-pip-conf,required=false \
+    if [ -s /run/secrets/codeartifact-pip-conf ]; then \
+      export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \
+      export UV_INDEX_SCALE_PYPI_USERNAME=aws; \
+      export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \
+        | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \
+    fi; \
     uv sync --no-dev
 
 ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH"

src/agentex/lib/cli/templates/default-openai-agents/Dockerfile.j2

@@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr
 
 WORKDIR /app/{{ project_path_from_build_root }}
 
+# Optional private index for Scale-internal packages such as sgp-obs, injected by the
+# control-plane broker (SGPINF-1568). Inert unless the secret is present.
+#
+# This variant installs from requirements.txt, so there is no pyproject.toml for uv to
+# read a named index out of; the credentialed URL is used directly and is deliberately
+# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory.
+#
 # Install the required Python packages
-RUN uv pip install --system -r requirements.txt
+RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \
+    if [ -s /run/secrets/codeartifact-pip-conf ]; then \
+      export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \
+    fi; \
+    uv pip install --system -r requirements.txt
 
 # Copy the project code
 COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project

src/agentex/lib/cli/templates/default-openai-agents/project/acp.py.j2

@@ -20,7 +20,7 @@ from dotenv import load_dotenv
 
 load_dotenv()
 
-from agents import Agent, Runner, function_tool, set_tracing_disabled
+from agents import Agent, Runner, function_tool, set_trace_processors
 
 from agentex.lib import adk
 from agentex.lib.types.acp import SendEventParams, CancelTaskParams, CreateTaskParams
@@ -34,10 +34,17 @@ from agentex.lib.core.harness.emitter import UnifiedEmitter
 from agentex.lib.adk import OpenAITurn
 from agentex.lib.core.tracing.tracing_processor_manager import add_tracing_processor_config
 
-# Disable the openai-agents SDK's native tracer so it doesn't ship traces to
-# api.openai.com using OPENAI_API_KEY (which may be a LiteLLM proxy key).
-# SGP tracing below still runs via the Agentex tracing manager.
-set_tracing_disabled(True)
+# Drop the openai-agents SDK's own exporter, so it can't ship traces to
+# api.openai.com using OPENAI_API_KEY (which may be a gateway/proxy key and would 401).
+#
+# Clearing the processor list rather than disabling tracing outright: disabling stops
+# spans being produced AT ALL, which silently starves any processor added later —
+# including the sgp-obs bridge the SDK installs when observability is on, so a Runner
+# turn would contribute no model spans. Clearing instead removes the OpenAI exporter
+# (which otherwise stays registered and is merely never fed) while leaving the
+# machinery alive for the bridge to attach to.
+# Agentex/SGP tracing still runs via the tracing manager.
+set_trace_processors([])
 
 logger = make_logger(__name__)