diff --git a/adk/pyproject.toml b/adk/pyproject.toml index b42b50e11..88125a8ce 100644 --- a/adk/pyproject.toml +++ b/adk/pyproject.toml @@ -65,6 +65,7 @@ dependencies = [ # agentex/lib/* uses `from typing import override` (3.12+) in 19 files. # The slim agentex-client keeps 3.11 support. requires-python = ">= 3.12,<4" + classifiers = [ "Typing :: Typed", "Intended Audience :: Developers", @@ -76,6 +77,23 @@ classifiers = [ "License :: OSI Approved :: Apache Software License", ] +# No `obs` extra, deliberately — do not add one for sgp-obs. +# +# sgp-obs is not on public PyPI (it is served from Scale's curated CodeArtifact +# mirror), and declaring it in [project.optional-dependencies] makes THIS repo's uv +# workspace unresolvable: `uv sync` re-locks, locking must resolve every declared +# optional dependency of every workspace member, and there is no way to exempt one. +# Measured: `uv lock --check`, `uv sync --all-extras`, plain `uv sync` with no extras, +# and `uv sync --all-extras --no-extra obs` all fail (`--no-extra` filters what is +# installed, not what is resolved); `uv lock` has no `--no-extra`; and +# `[tool.uv] override-dependencies` does not exempt it either. Only `--frozen` works, +# which would leave nobody able to re-lock this repo again. +# +# So the dependency is the AGENT's to declare — `sgp-obs[genai-auto,http,otlp]` +# against the mirror — and the SDK wires it when it is importable. See +# agentex/lib/core/observability/sgp_obs_setup.py; nothing imports sgp_obs outside a +# try, so a plain `pip install agentex-sdk` is unaffected either way. + [project.urls] Homepage = "https://github.com/scaleapi/scale-agentex-python" Repository = "https://github.com/scaleapi/scale-agentex-python" diff --git a/src/agentex/lib/cli/templates/PRIVATE_INDEX.md b/src/agentex/lib/cli/templates/PRIVATE_INDEX.md new file mode 100644 index 000000000..922107f9e --- /dev/null +++ b/src/agentex/lib/cli/templates/PRIVATE_INDEX.md @@ -0,0 +1,62 @@ +# The private package index in scaffold Dockerfiles + +Every scaffold Dockerfile mounts a build secret named `codeartifact-pip-conf`. It lets an agent +install Scale-internal packages — `sgp-obs`, for instance — that are not on public PyPI, without the +build holding any registry credential of its own. The control-plane broker mints a short-lived +CodeArtifact token per build and injects it as that secret. + +- Design: [Private Package Access for Customer Agents (PRD)](https://app.notion.com/p/Private-Package-Access-for-Customer-Agents-PRD-3ad904d6e6cb802cb091df1c25e230bc) +- Tracking: [SGPINF-1568](https://linear.app/scale-epd/issue/SGPINF-1568/provide-scale-internal-packages-to-agentex-agents-in-customer) + +## It is inert by default + +The mount is `required=false` and guarded by `[ -s ... ]`, so with no secret injected the build is +byte-identical to one without any of this. That covers every local build, every CI build, and every +agent that never opts in. An empty secret file is skipped too. + +## Opting in + +Add the index to the agent's `pyproject.toml`: + +```toml +[[tool.uv.index]] +name = "scale-pypi" +url = "" +default = true +``` + +The name must be exactly `scale-pypi`. uv applies `UV_INDEX_SCALE_PYPI_USERNAME` / +`UV_INDEX_SCALE_PYPI_PASSWORD` to the index of that name, so renaming it makes the credentials +silently stop applying. Setting `UV_INDEX_URL` instead does not authenticate a *named* index at +all, and the resolve fails with a 401. + +## Three things that are easy to get wrong + +**The token arrives percent-encoded.** The buildspec URL-encodes it to embed it in the pip config's +URL userinfo, so a token containing `+`, `/` or `=` arrives as `%2B`, `%2F`, `%3D`. The `uv sync` +templates decode it before exporting it as a password. Passing it through still-encoded sends a +different string and the resolve 401s. + +**The credential must not follow project-controlled configuration.** uv binds credentials by index +*name*, and the name-to-URL mapping would otherwise come from the agent's own `pyproject.toml` — so a +project that pointed `scale-pypi` at another host would receive the token. Verified against a local +server: the rogue host receives `Authorization: Basic aws:` and the real index is never +contacted. The templates therefore export `UV_INDEX` to re-bind the name to the URL the *broker* +supplied, which overrides whatever the project declared. With that in place the rogue host is never +contacted. The pinned URL carries no userinfo; the token still travels only in +`UV_INDEX_SCALE_PYPI_PASSWORD`. + +The case this defends is not a malicious agent author — they also write the Dockerfile and could read +the mounted secret directly. It is a *contributed* change to a project file, where a one-line URL edit +is far less conspicuous in review than an exfiltration command in a Dockerfile. + +**The two template variants work differently, deliberately.** + +| Template | Install step | How the credential is supplied | +| --- | --- | --- | +| `Dockerfile-uv.j2` | `uv sync` against the agent's `pyproject.toml` | Named index `scale-pypi`, pinned via `UV_INDEX`, token decoded into `UV_INDEX_SCALE_PYPI_PASSWORD` | +| `Dockerfile.j2` | `uv pip install -r requirements.txt` | No pyproject is present, so there is no named index to bind to. The credentialed URL is used directly via `UV_DEFAULT_INDEX` | + +The `requirements.txt` variant does **not** decode the token, and that is the point: it stays inside +the URL, already encoded for exactly that use. Decoding it there would corrupt it. It is also not +exposed to the redirection problem above, because the URL comes wholly from the injected secret. diff --git a/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 index 93d0f82d1..8a22d0f89 100644 --- a/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 @@ -34,7 +34,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -42,6 +55,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/default-claude-code/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-claude-code/Dockerfile.j2 index d714d96f9..3556f6dfd 100644 --- a/src/agentex/lib/cli/templates/default-claude-code/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-claude-code/Dockerfile.j2 @@ -33,8 +33,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 index 02860b9b9..b3c03c988 100644 --- a/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 @@ -34,7 +34,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -42,6 +55,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/default-codex/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-codex/Dockerfile.j2 index 1a8eb1484..c0b3fc385 100644 --- a/src/agentex/lib/cli/templates/default-codex/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-codex/Dockerfile.j2 @@ -33,8 +33,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 index dd3035f7b..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 @@ -30,7 +30,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +51,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/default-langgraph/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-langgraph/Dockerfile.j2 index 0395caf74..7f148e274 100644 --- a/src/agentex/lib/cli/templates/default-langgraph/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-langgraph/Dockerfile.j2 @@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 index dd3035f7b..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 @@ -30,7 +30,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +51,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile.j2 index 056d60b96..0a416aa38 100644 --- a/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile.j2 @@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/default-openai-agents/project/acp.py.j2 b/src/agentex/lib/cli/templates/default-openai-agents/project/acp.py.j2 index 66ee31243..ad8b6e41d 100644 --- a/src/agentex/lib/cli/templates/default-openai-agents/project/acp.py.j2 +++ b/src/agentex/lib/cli/templates/default-openai-agents/project/acp.py.j2 @@ -20,7 +20,7 @@ from dotenv import load_dotenv load_dotenv() -from agents import Agent, Runner, function_tool, set_tracing_disabled +from agents import Agent, Runner, function_tool, set_trace_processors from agentex.lib import adk from agentex.lib.types.acp import SendEventParams, CancelTaskParams, CreateTaskParams @@ -34,10 +34,17 @@ from agentex.lib.core.harness.emitter import UnifiedEmitter from agentex.lib.adk import OpenAITurn from agentex.lib.core.tracing.tracing_processor_manager import add_tracing_processor_config -# Disable the openai-agents SDK's native tracer so it doesn't ship traces to -# api.openai.com using OPENAI_API_KEY (which may be a LiteLLM proxy key). -# SGP tracing below still runs via the Agentex tracing manager. -set_tracing_disabled(True) +# Drop the openai-agents SDK's own exporter, so it can't ship traces to +# api.openai.com using OPENAI_API_KEY (which may be a gateway/proxy key and would 401). +# +# Clearing the processor list rather than disabling tracing outright: disabling stops +# spans being produced AT ALL, which silently starves any processor added later — +# including the sgp-obs bridge the SDK installs when observability is on, so a Runner +# turn would contribute no model spans. Clearing instead removes the OpenAI exporter +# (which otherwise stays registered and is merely never fed) while leaving the +# machinery alive for the bridge to attach to. +# Agentex/SGP tracing still runs via the tracing manager. +set_trace_processors([]) logger = make_logger(__name__) diff --git a/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 index dd3035f7b..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 @@ -30,7 +30,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +51,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile.j2 index 0395caf74..7f148e274 100644 --- a/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile.j2 @@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 index dd3035f7b..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 @@ -30,7 +30,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +51,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/default/Dockerfile.j2 b/src/agentex/lib/cli/templates/default/Dockerfile.j2 index 0395caf74..7f148e274 100644 --- a/src/agentex/lib/cli/templates/default/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default/Dockerfile.j2 @@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 index 93d0f82d1..8a22d0f89 100644 --- a/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 @@ -34,7 +34,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -42,6 +55,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile.j2 index 6cdc70799..cd0338d18 100644 --- a/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile.j2 @@ -33,8 +33,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 index 02860b9b9..b3c03c988 100644 --- a/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 @@ -34,7 +34,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -42,6 +55,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync-codex/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-codex/Dockerfile.j2 index afa4470d9..79293756d 100644 --- a/src/agentex/lib/cli/templates/sync-codex/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-codex/Dockerfile.j2 @@ -33,8 +33,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 index dd3035f7b..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 @@ -30,7 +30,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +51,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile.j2 index 4d9f41d45..d0c204e47 100644 --- a/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile.j2 @@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 index dd3035f7b..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 @@ -30,7 +30,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +51,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile.j2 index 4d9f41d45..d0c204e47 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile.j2 @@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/project/agent.py.j2 b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/project/agent.py.j2 index 07546bffb..315c5a6ae 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/project/agent.py.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/project/agent.py.j2 @@ -15,7 +15,7 @@ from __future__ import annotations from datetime import datetime -from agents import Runner, set_tracing_disabled +from agents import Runner, set_trace_processors from agents.sandbox import SandboxAgent, SandboxRunConfig from agents.run_config import RunConfig from agents.sandbox.sandboxes.unix_local import ( @@ -25,10 +25,17 @@ from agents.sandbox.sandboxes.unix_local import ( from project.tools import get_capabilities -# Disable the openai-agents SDK's native tracer so it doesn't ship traces to -# api.openai.com using OPENAI_API_KEY (which may be a gateway/proxy key and would -# 401). Agentex tracing still runs via the tracing manager configured in acp.py. -set_tracing_disabled(True) +# Drop the openai-agents SDK's own exporter, so it can't ship traces to +# api.openai.com using OPENAI_API_KEY (which may be a gateway/proxy key and would 401). +# +# Clearing the processor list rather than disabling tracing outright: disabling stops +# spans being produced AT ALL, which silently starves any processor added later — +# including the sgp-obs bridge the SDK installs when observability is on, so a Runner +# turn would contribute no model spans. Clearing instead removes the OpenAI exporter +# (which otherwise stays registered and is merely never fed) while leaving the +# machinery alive for the bridge to attach to. +# Agentex/SGP tracing still runs via the tracing manager. +set_trace_processors([]) MODEL_NAME = "gpt-4o-mini" INSTRUCTIONS = """You are a local sandbox assistant. diff --git a/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 index dd3035f7b..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 @@ -30,7 +30,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +51,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile.j2 index 4d9f41d45..d0c204e47 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile.j2 @@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync-openai-agents/project/acp.py.j2 b/src/agentex/lib/cli/templates/sync-openai-agents/project/acp.py.j2 index 41029f2ce..07849e81d 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents/project/acp.py.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents/project/acp.py.j2 @@ -13,12 +13,19 @@ from agentex.types.task_message_update import TaskMessageUpdate, StreamTaskMessa from agentex.types.task_message_content import TaskMessageContent from agentex.types.text_content import TextContent from agentex.lib.utils.logging import make_logger -from agents import Agent, Runner, RunConfig, function_tool, set_tracing_disabled - -# Disable the openai-agents SDK's native tracer so it doesn't ship traces to -# api.openai.com using OPENAI_API_KEY (which may be a LiteLLM proxy key). -# SGP tracing below still runs via the Agentex tracing manager. -set_tracing_disabled(True) +from agents import Agent, Runner, RunConfig, function_tool, set_trace_processors + +# Drop the openai-agents SDK's own exporter, so it can't ship traces to +# api.openai.com using OPENAI_API_KEY (which may be a gateway/proxy key and would 401). +# +# Clearing the processor list rather than disabling tracing outright: disabling stops +# spans being produced AT ALL, which silently starves any processor added later — +# including the sgp-obs bridge the SDK installs when observability is on, so a Runner +# turn would contribute no model spans. Clearing instead removes the OpenAI exporter +# (which otherwise stays registered and is merely never fed) while leaving the +# machinery alive for the bridge to attach to. +# Agentex/SGP tracing still runs via the tracing manager. +set_trace_processors([]) logger = make_logger(__name__) diff --git a/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 index dd3035f7b..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 @@ -30,7 +30,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +51,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile.j2 index 4d9f41d45..d0c204e47 100644 --- a/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile.j2 @@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 index dd3035f7b..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 @@ -30,7 +30,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +51,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync/Dockerfile.j2 index 4d9f41d45..d0c204e47 100644 --- a/src/agentex/lib/cli/templates/sync/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync/Dockerfile.j2 @@ -29,8 +29,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 index f8746c573..1665bceb1 100644 --- a/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 @@ -42,7 +42,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -50,6 +63,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile.j2 index 225863607..1297b7bd7 100644 --- a/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile.j2 @@ -41,8 +41,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 index 7e31387fa..41d83e31c 100644 --- a/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 @@ -42,7 +42,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -50,6 +63,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/temporal-codex/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-codex/Dockerfile.j2 index 0ae4e2079..d77d8073f 100644 --- a/src/agentex/lib/cli/templates/temporal-codex/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-codex/Dockerfile.j2 @@ -41,8 +41,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 index 6746869df..56b4d949c 100644 --- a/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 @@ -36,7 +36,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -44,6 +57,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile.j2 index ba47485a9..5bb133a22 100644 --- a/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile.j2 @@ -35,8 +35,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 index 0d9801016..a674d7d35 100644 --- a/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 @@ -36,7 +36,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -44,6 +57,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile.j2 index 4c1798c42..a9a63757d 100644 --- a/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile.j2 @@ -35,8 +35,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/temporal-openai-agents/project/workflow.py.j2 b/src/agentex/lib/cli/templates/temporal-openai-agents/project/workflow.py.j2 index af8b7a299..1b2ae547c 100644 --- a/src/agentex/lib/cli/templates/temporal-openai-agents/project/workflow.py.j2 +++ b/src/agentex/lib/cli/templates/temporal-openai-agents/project/workflow.py.j2 @@ -10,12 +10,19 @@ from agentex.lib.core.temporal.types.workflow import SignalName from agentex.lib.utils.logging import make_logger from agentex.types.text_content import TextContent from agentex.lib.environment_variables import EnvironmentVariables -from agents import Agent, Runner, set_tracing_disabled - -# Disable the openai-agents SDK's native tracer so it doesn't ship traces to -# api.openai.com using OPENAI_API_KEY (which may be a LiteLLM proxy key). -# SGP tracing below still runs via the Agentex tracing manager. -set_tracing_disabled(True) +from agents import Agent, Runner, set_trace_processors + +# Drop the openai-agents SDK's own exporter, so it can't ship traces to +# api.openai.com using OPENAI_API_KEY (which may be a gateway/proxy key and would 401). +# +# Clearing the processor list rather than disabling tracing outright: disabling stops +# spans being produced AT ALL, which silently starves any processor added later — +# including the sgp-obs bridge the SDK installs when observability is on, so a Runner +# turn would contribute no model spans. Clearing instead removes the OpenAI exporter +# (which otherwise stays registered and is merely never fed) while leaving the +# machinery alive for the bridge to attach to. +# Agentex/SGP tracing still runs via the tracing manager. +set_trace_processors([]) from agentex.lib.core.temporal.plugins.openai_agents.hooks.hooks import TemporalStreamingHooks from pydantic import BaseModel diff --git a/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 index 0d9801016..a674d7d35 100644 --- a/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 @@ -36,7 +36,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -44,6 +57,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile.j2 index 4c1798c42..a9a63757d 100644 --- a/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile.j2 @@ -35,8 +35,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 index 0d9801016..a674d7d35 100644 --- a/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 @@ -36,7 +36,20 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -44,6 +57,13 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/temporal/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal/Dockerfile.j2 index 4c1798c42..a9a63757d 100644 --- a/src/agentex/lib/cli/templates/temporal/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal/Dockerfile.j2 @@ -35,8 +35,19 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. +# +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/tests/__init__.py b/src/agentex/lib/cli/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/agentex/lib/cli/tests/test_template_tracing.py b/src/agentex/lib/cli/tests/test_template_tracing.py new file mode 100644 index 000000000..adb76fd08 --- /dev/null +++ b/src/agentex/lib/cli/tests/test_template_tracing.py @@ -0,0 +1,57 @@ +"""The openai-agents scaffolds must not disable tracing outright. + +`set_tracing_disabled(True)` stops openai-agents producing spans AT ALL, which +silently starves any processor registered later — including the sgp-obs bridge the SDK +installs when observability is on. The bridge still reports itself installed, so a +Runner turn contributes no model spans and nothing says why. + +Measured against a spy processor: + + set_tracing_disabled(True) -> processors ['BatchTraceProcessor', 'Spy'], spy saw 0 + set_trace_processors([]) -> processors ['Spy'], spy saw 1 + +Note the first row: disabling tracing leaves the OpenAI exporter REGISTERED, merely +never fed. Clearing the list actually removes it, so the replacement is strictly better +at the thing the original was trying to do — keep traces away from api.openai.com. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +TEMPLATES = Path(__file__).resolve().parents[1] / "templates" + + +def _templates_using_agents_tracing() -> list[Path]: + return sorted( + p for p in TEMPLATES.rglob("*.j2") if "set_trace_processors" in p.read_text() + ) + + +def test_some_templates_were_found(): + """Guards the glob itself: if the templates move, the assertions below would + vacuously pass on an empty list.""" + assert _templates_using_agents_tracing(), f"no templates found under {TEMPLATES}" + + +@pytest.mark.parametrize( + "template", _templates_using_agents_tracing(), ids=lambda p: p.parent.parent.name +) +class TestOpenAIAgentsScaffolds: + def test_does_not_disable_tracing(self, template: Path): + text = template.read_text() + assert "set_tracing_disabled(" not in text, ( + f"{template} disables openai-agents tracing, which starves the sgp-obs bridge" + ) + + def test_clears_the_processor_list_instead(self, template: Path): + assert "set_trace_processors([])" in template.read_text() + + def test_imports_what_it_calls(self, template: Path): + text = template.read_text() + assert "set_trace_processors" in text.split("\n\n")[0] or any( + "import" in line and "set_trace_processors" in line + for line in text.splitlines() + ), f"{template} calls set_trace_processors without importing it" diff --git a/src/agentex/lib/core/adapters/llm/_genai_metrics.py b/src/agentex/lib/core/adapters/llm/_genai_metrics.py new file mode 100644 index 000000000..3c2293cd6 --- /dev/null +++ b/src/agentex/lib/core/adapters/llm/_genai_metrics.py @@ -0,0 +1,133 @@ +"""GenAI metrics for the litellm gateway, via ``sgp_obs.metrics.genai.call()``. + +Why the SDK does this rather than leaving it to zero-code instrumentation: + +Most model calls in the fleet reach the wire through the ``openai`` client, and for +those, patching that one client covers everything with no code — ``Runner.run``, the +ADK's openai provider, and litellm pointed at an OpenAI-compatible proxy. The client +patch cannot help in two situations, and this gateway hits both: + +1. **litellm routing natively** to Anthropic, Bedrock, Vertex or Azure never touches + the ``openai`` client, so nothing records it at all. +2. Even in proxy mode, the patch sits *inside* the OpenAI client, so it reports + ``gen_ai.provider.name="openai"`` — the protocol. It cannot know that the caller + asked for ``claude-sonnet-4``. This gateway chose the vendor, so it can say so. + +``transport=`` resolves the overlap between the two: when the call is going out over +the OpenAI client, we name that, and ``call()`` stands down if the client instrumentor +is already recording. When litellm routes natively there is no such overlap, so we +record. That decision is made per call, from the model string, in +:func:`_split_model`. + +Everything here is fail-open: sgp-obs is an optional dependency and a telemetry problem +must never fail a model call. If the import fails, :func:`inference_call` returns an +object that records nothing and costs nothing. +""" + +from __future__ import annotations + +from typing import Any + +from agentex.lib.utils.logging import make_logger + +logger = make_logger(__name__) + +# litellm's directive for "send this to the configured OpenAI-compatible proxy". It is +# a routing instruction, not a vendor, so it is stripped before reading the vendor. +_PROXY_PREFIX = "litellm_proxy/" + +# A bare model name with no "/" prefix is OpenAI, per litellm's own default. +_DEFAULT_VENDOR = "openai" + +_warned = False + + +def _split_model(model: str) -> tuple[str, bool]: + """``(vendor, goes_out_over_the_openai_client)`` for a litellm model string. + + ``"litellm_proxy/anthropic/claude-sonnet-4"`` -> ``("anthropic", True)`` + ``"anthropic/claude-sonnet-4"`` -> ``("anthropic", False)`` + ``"gpt-4o"`` -> ``("openai", True)`` + + A bare name is OpenAI, and litellm reaches OpenAI through the ``openai`` + client, so the client instrumentor already sees it and we stand down. + """ + proxied = model.startswith(_PROXY_PREFIX) + rest = model[len(_PROXY_PREFIX):] if proxied else model + vendor = rest.split("/", 1)[0] if "/" in rest else _DEFAULT_VENDOR + # Proxy mode always leaves over the OpenAI client. So does a native openai/* call. + return (vendor or _DEFAULT_VENDOR), proxied or vendor == _DEFAULT_VENDOR + + +def resolve_model(args: tuple[Any, ...], kwargs: dict[str, Any]) -> str: + """The model for a litellm call, whether it arrived by keyword or positionally. + + ``litellm.acompletion`` takes ``model`` as its FIRST positional argument, and the + gateway forwards ``*args`` untouched, so ``gateway.acompletion("anthropic/claude- + sonnet-4", messages)`` is a legal call that puts the model in ``args[0]``. + + Reading only ``kwargs`` there does not merely mislabel the vendor, it loses the + measurement: an empty model resolves to the default vendor "openai", which sets + ``transport=OPENAI``, which makes ``call()`` stand down for the OpenAI client + instrumentor — while litellm routes natively to Anthropic and never touches that + client. Nothing records it and nothing says so. + """ + model = kwargs.get("model") + if not model and args: + model = args[0] + # Positional args are forwarded verbatim, so args[0] is whatever the caller passed; + # only a string can be a litellm model name. + return model if isinstance(model, str) else "" + + +def inference_call(kwargs: dict[str, Any], args: tuple[Any, ...] = ()) -> Any: + """Begin recording one litellm call. Never raises, never returns None.""" + try: + # See sgp_obs_setup.py: optional, not publicly installable, absent in CI. + from sgp_obs.metrics import genai # type: ignore[import-not-found] + except Exception: + global _warned + if not _warned: + _warned = True + logger.debug( + "sgp-obs is not available; GenAI metrics are off for litellm calls" + ) + return _NULL_CALL + + try: + model = resolve_model(args, kwargs) + vendor, over_openai_client = _split_model(model) + return genai.call( + provider=vendor, + operation=genai.CHAT, + model=model, + # litellm normalises every vendor's response onto the OpenAI shape, so one + # parser reads them all — which is exactly what `spec` separates from the + # `provider` label. + spec=genai.OPENAI_SPEC, + transport=genai.OPENAI if over_openai_client else "", + ) + except Exception: + logger.debug("could not start a GenAI metrics record", exc_info=True) + return _NULL_CALL + + +class _NullCall: + """What call sites get when sgp-obs is absent. Records nothing, costs nothing.""" + + def observe(self, response: Any) -> Any: + return response + + # Underscored like __aexit__'s params below: present for parity with the real + # sgp-obs call object, never read here. + def failed(self, _error: BaseException) -> None: + return + + async def __aenter__(self) -> "_NullCall": + return self + + async def __aexit__(self, _exc_type: Any, _exc: Any, _tb: Any) -> bool: + return False # never suppress the caller's exception + + +_NULL_CALL = _NullCall() diff --git a/src/agentex/lib/core/adapters/llm/adapter_litellm.py b/src/agentex/lib/core/adapters/llm/adapter_litellm.py index 7935f5f49..8fb1602aa 100644 --- a/src/agentex/lib/core/adapters/llm/adapter_litellm.py +++ b/src/agentex/lib/core/adapters/llm/adapter_litellm.py @@ -6,6 +6,7 @@ from agentex.lib.utils.logging import make_logger from agentex.lib.types.llm_messages import Completion from agentex.lib.core.adapters.llm.port import LLMGateway +from agentex.lib.core.adapters.llm._genai_metrics import inference_call logger = make_logger(__name__) @@ -36,9 +37,13 @@ async def acompletion(self, *args, **kwargs) -> Completion: "Please use self.acompletion_stream instead of self.acompletion to stream responses" ) - # Return a single completion for non-streaming - response = await llm.acompletion(*args, **kwargs) - return Completion.model_validate(response) + # `async with`, not try/except: asyncio.CancelledError is a BaseException, so a + # caller that disappears mid-flight would skip an `except Exception` handler and + # the record would be silently dropped. + async with inference_call(kwargs, args) as call: + # Return a single completion for non-streaming + response = call.observe(await llm.acompletion(*args, **kwargs)) + return Completion.model_validate(response) @override async def acompletion_stream( @@ -47,5 +52,11 @@ async def acompletion_stream( if not kwargs.get("stream"): raise ValueError("To use streaming, please set stream=True in the kwargs") - async for chunk in await llm.acompletion(*args, **kwargs): # type: ignore[misc] - yield Completion.model_validate(chunk) + async with inference_call(kwargs, args) as call: + # observe() takes ownership of the stream and yields the same chunks, so it + # can read time-to-first-chunk and the token totals off the last chunk. + # Wrapping only the `await` would return before the first chunk arrived and + # record zero tokens for every streamed call. + stream = call.observe(await llm.acompletion(*args, **kwargs)) + async for chunk in stream: # type: ignore[misc] + yield Completion.model_validate(chunk) diff --git a/src/agentex/lib/core/adapters/llm/tests/__init__.py b/src/agentex/lib/core/adapters/llm/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/agentex/lib/core/adapters/llm/tests/test_genai_metrics.py b/src/agentex/lib/core/adapters/llm/tests/test_genai_metrics.py new file mode 100644 index 000000000..b4276fdb1 --- /dev/null +++ b/src/agentex/lib/core/adapters/llm/tests/test_genai_metrics.py @@ -0,0 +1,174 @@ +"""Tests for ``agentex.lib.core.adapters.llm._genai_metrics``. + +The important property is the one that holds in every environment today: with +``sgp-obs`` absent, :func:`inference_call` must hand back something the litellm +gateway can drive as an async context manager, whose ``observe()`` returns the +response untouched and which never swallows the caller's exception. That is the +path every agent without the ``obs`` extra takes on every model call, so a +regression here breaks model calls rather than just losing a metric. +""" + +from __future__ import annotations + +import sys +import builtins + +import pytest + +from agentex.lib.core.adapters.llm import _genai_metrics +from agentex.lib.core.adapters.llm._genai_metrics import ( + _split_model, + resolve_model, + inference_call, +) + + +class TestSplitModel: + """``(vendor, goes_out_over_the_openai_client)``. The boolean decides whether + ``call()`` stands down for the OpenAI client instrumentor or records itself, so + getting it wrong either double-counts a call or loses it.""" + + @pytest.mark.parametrize( + ("model", "vendor", "over_openai_client"), + [ + # Proxy mode: litellm sends this to an OpenAI-compatible proxy over the + # openai client, but the caller asked for a non-OpenAI vendor. + ("litellm_proxy/anthropic/claude-sonnet-4", "anthropic", True), + ("litellm_proxy/gpt-4o", "openai", True), + # Native routing: litellm's own handler, no openai client involved. + ("anthropic/claude-sonnet-4", "anthropic", False), + ("bedrock/anthropic.claude-v2", "bedrock", False), + ("vertex_ai/gemini-2.0-flash", "vertex_ai", False), + # A bare name is OpenAI per litellm's default, and reaches OpenAI + # through the openai client — so the instrumentor already sees it. + ("gpt-4o", "openai", True), + ("openai/gpt-4o", "openai", True), + ], + ) + def test_vendor_and_transport(self, model, vendor, over_openai_client): + assert _split_model(model) == (vendor, over_openai_client) + + def test_empty_model_does_not_raise(self): + """kwargs.get("model") is "" when a caller passes model positionally. + Falling back to litellm's own default is right, and must not blow up.""" + assert _split_model("") == ("openai", True) + + +class TestFailsOpenWithoutSgpObs: + @staticmethod + def _hide_sgp_obs(monkeypatch): + for name in [m for m in sys.modules if m.startswith("sgp_obs")]: + monkeypatch.delitem(sys.modules, name, raising=False) + real_import = builtins.__import__ + + def no_sgp_obs(name, *args, **kwargs): + if name == "sgp_obs" or name.startswith("sgp_obs."): + raise ImportError("No module named 'sgp_obs'") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", no_sgp_obs) + # The "already warned" latch is module state; reset so the path is exercised. + monkeypatch.setattr(_genai_metrics, "_warned", False) + + def test_returns_a_usable_recorder_not_none(self, monkeypatch): + self._hide_sgp_obs(monkeypatch) + assert inference_call({"model": "gpt-4o"}) is _genai_metrics._NULL_CALL + + async def test_observe_returns_the_response_unchanged(self, monkeypatch): + """The gateway does `call.observe(await acompletion(...))`, so an observe() + that returned None would turn every completion into None.""" + self._hide_sgp_obs(monkeypatch) + sentinel = object() + async with inference_call({"model": "gpt-4o"}) as call: + assert call.observe(sentinel) is sentinel + + async def test_does_not_suppress_the_callers_exception(self, monkeypatch): + """__aexit__ must return falsey. Suppressing here would make a failed model + call look like a successful one that returned nothing.""" + self._hide_sgp_obs(monkeypatch) + with pytest.raises(ValueError, match="upstream"): + async with inference_call({"model": "gpt-4o"}): + raise ValueError("upstream blew up") + + async def test_cancellation_still_propagates(self, monkeypatch): + """CancelledError is a BaseException; the `async with` in the gateway exists + so a disappearing caller is not silently dropped.""" + import asyncio + + self._hide_sgp_obs(monkeypatch) + with pytest.raises(asyncio.CancelledError): + async with inference_call({"model": "gpt-4o"}): + raise asyncio.CancelledError() + + def test_a_broken_sgp_obs_does_not_break_a_model_call(self, monkeypatch): + """Not just ImportError: anything raised while starting a record must fall + back to the null recorder.""" + module = type(sys)("sgp_obs.metrics") + genai = type(sys)("genai") + + def exploding(**_kwargs): + raise RuntimeError("sgp-obs internals changed") + + genai.call = exploding + genai.CHAT = "chat" + genai.OPENAI_SPEC = "openai" + genai.OPENAI = "openai" + module.genai = genai + monkeypatch.setitem(sys.modules, "sgp_obs", type(sys)("sgp_obs")) + monkeypatch.setitem(sys.modules, "sgp_obs.metrics", module) + assert inference_call({"model": "gpt-4o"}) is _genai_metrics._NULL_CALL + + +class TestResolveModel: + """litellm takes `model` as its FIRST positional argument and the gateway forwards + *args untouched, so a positional call is legal and must still be measured. + + Reading only kwargs does not merely mislabel the vendor: an empty model resolves to + the default vendor "openai", which sets transport=OPENAI, which makes call() stand + down for the OpenAI client instrumentor — while litellm routes natively to Anthropic + and never touches that client. Nothing records it and nothing says so. + """ + + def test_keyword_model(self): + assert resolve_model((), {"model": "gpt-4o"}) == "gpt-4o" + + def test_positional_model(self): + assert resolve_model(("anthropic/claude-sonnet-4",), {}) == "anthropic/claude-sonnet-4" + + def test_keyword_wins_over_positional(self): + """litellm itself would reject both, but if it ever resolved one, the keyword is + the explicit intent.""" + assert resolve_model(("a/b",), {"model": "c/d"}) == "c/d" + + def test_no_model_at_all(self): + assert resolve_model((), {}) == "" + + def test_a_non_string_first_arg_is_not_a_model(self): + """*args is forwarded verbatim, so args[0] is whatever the caller passed.""" + assert resolve_model(([{"role": "user"}],), {}) == "" + + def test_positional_native_vendor_does_not_stand_down(self, monkeypatch): + """The regression this guards: a positional Anthropic model must be recorded by + the gateway, because nothing else will.""" + seen = {} + + class _Genai: + CHAT = "chat" + OPENAI_SPEC = "openai" + OPENAI = "openai" + + @staticmethod + def call(**kwargs): + seen.update(kwargs) + return _genai_metrics._NULL_CALL + + module = type(sys)("sgp_obs.metrics") + module.genai = _Genai + monkeypatch.setitem(sys.modules, "sgp_obs", type(sys)("sgp_obs")) + monkeypatch.setitem(sys.modules, "sgp_obs.metrics", module) + + inference_call({}, ("anthropic/claude-sonnet-4",)) + assert seen["model"] == "anthropic/claude-sonnet-4" + assert seen["provider"] == "anthropic" + # Empty transport == "no OpenAI-client overlap, so record it here". + assert seen["transport"] == "" diff --git a/src/agentex/lib/core/observability/sgp_obs_setup.py b/src/agentex/lib/core/observability/sgp_obs_setup.py new file mode 100644 index 000000000..5b76766f5 --- /dev/null +++ b/src/agentex/lib/core/observability/sgp_obs_setup.py @@ -0,0 +1,350 @@ +"""Optional sgp-obs wiring: traces, metrics and logs, switched on by environment. + +Why this lives in the SDK rather than in each agent: the fleet is ~147 agent repos, +and their deployments pin an exact SDK version. Doing the wiring here means an agent +adopts observability by installing ``sgp-obs`` and setting environment, instead of +carrying the wiring code — including the two parts that are easy to get wrong and +fail silently (where ``init()`` is called from, and flushing on the way out). + +``sgp-obs`` is NOT declared as a dependency or an extra of this package. It is not on +public PyPI, and declaring it would make this repo's own uv workspace unresolvable: +``uv sync`` re-locks, locking must resolve every declared optional dependency, and +neither ``--no-extra`` nor ``[tool.uv] override-dependencies`` exempts one. So the +contract is inverted — an agent declares ``sgp-obs[genai-auto,http,otlp]`` itself, +against Scale's curated mirror, and this module wires it if it is importable. Nothing +here imports ``sgp_obs`` outside a ``try``, so a plain ``pip install agentex-sdk`` +behaves exactly as it did before this module existed. + +TWO gates, both of which must pass before anything is recorded: + +1. ``sgp-obs`` must be importable. If it is not, this returns ``"not_installed"``. +2. The environment must ask for it. As of sgp-obs 0.16.0 every signal is opt-in + TWICE: the master switch ``SGP_OBS_ENABLED=true``, AND that signal's + ``*_DISABLED`` variable set to an explicit ``false``. An unset ``*_DISABLED`` + leaves the signal OFF. So the master switch on its own wires nothing at all — + measured on 0.16.0, ``SGP_OBS_ENABLED=true`` alone returns zero handles. All + three signals together need:: + + SGP_OBS_ENABLED=true + SGP_METRICS_DISABLED=false + SGP_TRACES_DISABLED=false + SGP_LOGS_DISABLED=false + + That inverts the advice written against 0.15.0, where traces came on with the + master switch and had to be turned off. This module does not second-guess the + gate — it calls ``init()`` and reports which signals came back — but it does + warn when the master switch is on and nothing wired, because that combination + is otherwise completely silent. + +Metrics additionally need an OTLP endpoint. sgp-obs never builds a MeterProvider +from nothing; in a cluster the OTel Operator's auto-instrumentation normally +supplies one, and agent pods get no injection, so ``OTEL_EXPORTER_OTLP_ENDPOINT`` +has to be on the pod spec. + +Fail-open is absolute: this is telemetry, and no failure here may stop an agent from +starting or serving. Every path returns a status string instead of raising. +""" + +from __future__ import annotations + +import os +from typing import Any + +from agentex.lib.utils.logging import ( + make_logger, + _reset_for_tests as _logging_reset_for_tests, + route_agentex_loggers_to_root, +) + +logger = make_logger(__name__) + +_status: str | None = None + +# sgp-obs' own truthy set (sgp_obs.env._TRUTHY), so "is the master switch on?" is +# answered the same way here as in the library deciding whether to wire. +_TRUTHY = {"1", "true", "yes", "on"} + +# The logs-profile selector. The SDK knows the runtime is agentex; an agent author +# would have to know to pass it. It stamps agent_id (from AGENT_ID) and task_id (from +# the SDK's streaming contextvar) onto every log record. +_SOURCE = "agentex" + + +def _master_switch_on() -> bool: + return (os.getenv("SGP_OBS_ENABLED") or "").strip().lower() in _TRUTHY + + +def init_sgp_obs(app: Any = None) -> str: + """Wire sgp-obs if it is installed and enabled. Returns a status; never raises. + + Statuses: ``"not_installed"``, ``"disabled"``, ``"wired:"``, ``"error"``. + + ``app`` is the ACP server. Passing it is what adds ``http.server.*`` for the + agent's own entry point — without it the agent is observable only from the + model call outwards, and its own latency and error rate cannot be alerted on. + It is also what installs the trace-context ingress middleware, so an incoming + ``traceparent`` continues into the agent's spans rather than starting a new trace. + """ + global _status + if _status is not None: + # init() is not meant to run twice, and a Temporal worker plus an ACP + # server can both reach this in one process. + return _status + + try: + # Not resolvable in a normal env: sgp-obs is not a dependency of this + # package and is not on public PyPI. That is the case this branch exists for. + import sgp_obs # type: ignore[import-not-found] + except ImportError: + if _master_switch_on(): + # The operator asked for observability and the package is absent. Silence + # here is the worst outcome, so say what is missing and how to fix it. + logger.warning( + "SGP_OBS_ENABLED is set but sgp-obs is not installed, so no telemetry " + "will be produced. Add sgp-obs[genai-auto,http,otlp] to this agent's " + "dependencies (it resolves from Scale's curated mirror, not public PyPI)." + ) + _status = "not_installed" + return _status + except Exception: # pragma: no cover - a broken install must not stop startup + logger.debug("sgp-obs import failed unexpectedly", exc_info=True) + _status = "error" + return _status + + try: + handles = sgp_obs.init( + app=app, + # Fills OTEL_SERVICE_NAME only when the deployment left it unset or + # blank; the deployment always outranks this. Without either, every + # signal is attributed to service.name="unknown". + service_name=(os.getenv("AGENT_NAME") or "").strip() or None, + source=_SOURCE, + ) + except Exception: # pragma: no cover - sgp_obs.init is itself fail-open + # One deliberate exception to its fail-open rule: under the standard CI + # variable, any logs misconfiguration raises so a build cannot pass while + # logging is broken. Swallowed here regardless — an agent must still serve. + logger.warning("sgp-obs initialization failed; continuing without it", exc_info=True) + _status = "error" + return _status + + if not handles: + if _master_switch_on(): + # 0.16.0's double opt-in: the master switch alone wires nothing, and + # sgp-obs says nothing about it. Name the variables that are missing. + logger.warning( + "SGP_OBS_ENABLED is set but no sgp-obs signal is enabled, so nothing " + "will be exported. Each signal is opt-in separately: set " + "SGP_METRICS_DISABLED=false, SGP_TRACES_DISABLED=false and " + "SGP_LOGS_DISABLED=false for the signals you want. An unset " + "*_DISABLED leaves that signal off." + ) + # Otherwise expected, and the default: an agent with sgp-obs installed still + # records nothing until someone sets the environment. + _status = "disabled" + return _status + + if "logs" in handles: + _hand_agentex_logging_to_the_pipeline() + + if "traces" in handles: + _install_openai_agents_bridge() + _warn_if_correlation_backend_mismatched() + + _status = "wired:" + ",".join(sorted(handles)) + logger.info("sgp-obs wired (%s)", _status) + return _status + + +def _hand_agentex_logging_to_the_pipeline() -> None: + """Stop agentex's own loggers printing a second, ungoverned copy of every record. + + ``agentex.lib.utils.logging.make_logger`` attaches a handler to each module's own + (leaf) logger. sgp-obs' logs pipeline replaces the handlers on the ROOT logger and + deliberately leaves named loggers alone, because a named logger's handler may be + there on purpose. The two are individually correct and together print everything + twice: once in agentex's plain-text format from the leaf, once as pipeline JSON + from root. Measured on sgp-obs 0.16.0, one ``logger.info()`` gave two stdout lines, + and sgp-obs' boot warning named 63 loggers. + + The duplicate is not merely redundant: it is emitted before the pipeline's filters, + so it carries no ``agent_id``/``task_id``, is not governed by the allowlist, and is + not truncated. + + Only agentex's loggers are handed over — see + :func:`~agentex.lib.utils.logging.route_agentex_loggers_to_root` for why by prefix, + why ``capture_loggers=`` is not the mechanism, and why a third party's handler is + left where it is. + """ + try: + cleared = route_agentex_loggers_to_root() + except Exception: # pragma: no cover - telemetry must never break startup + logger.debug("could not hand agentex logging to the sgp-obs pipeline", exc_info=True) + return + + if cleared: + # sgp-obs has already logged its "bypass log governance" warning by this point, + # naming loggers this call has just fixed. Say so, or the two lines read as a + # contradiction to whoever is looking at the pod's first second of output. + logger.info( + "routed %d agentex logger(s) through the sgp-obs logs pipeline; any " + "'bypass log governance' warning above that names agentex.* loggers was " + "emitted before this ran and no longer applies to them", + cleared, + ) + + +def _install_openai_agents_bridge() -> bool: + """Register sgp-obs' openai-agents trace processor, so a ``Runner`` turn produces + logical model-operation spans. + + This is the one piece of traces wiring ``sgp_obs.init()`` does NOT do for itself. + Measured on 0.16.0 after a plain ``init()`` with the traces signal on: + + GenAI attempt span processor installed + litellm logical adapter installed + httpx / aiohttp egress instrumented + openai-agents bridge NOT installed + + which is why the obs-test agents each carry a hand-written bootstrap that calls it. + It matters more than the others here: roughly 83% of model-calling agents reach the + model through the openai-agents ``Runner``, so without this the dominant path + contributes no logical spans and "traces on" looks like it does nothing. + + Unconditional because ``openai-agents`` is a hard dependency of this SDK, so the + ``agents`` package is importable in every agent. The call is idempotent and returns + False rather than raising when the SDK is somehow absent. + """ + try: + from sgp_obs.traces import install_openai_agents_bridge # type: ignore[import-not-found] + + installed = bool(install_openai_agents_bridge()) + if installed: + logger.debug("sgp-obs openai-agents bridge installed") + _warn_if_openai_agents_tracing_disabled() + else: + # Only reachable if `agents` is not importable, which should not happen + # while openai-agents is a hard dependency — so say so rather than shrug. + logger.warning( + "sgp-obs openai-agents bridge did not install; Runner turns will " + "produce no logical model-operation spans." + ) + return installed + except Exception: # pragma: no cover - telemetry must never break startup + logger.debug("sgp-obs openai-agents bridge unavailable", exc_info=True) + return False + + +def _warn_if_openai_agents_tracing_disabled() -> None: + """Warn when the bridge is installed but openai-agents tracing is switched off. + + ``install_openai_agents_bridge()`` returns True as soon as it registers itself as a + trace processor — it cannot tell whether the provider will ever feed it. If the + agent called ``set_tracing_disabled(True)``, no spans are produced at all, so the + bridge is registered and permanently idle, and nothing says so. + + That is not hypothetical: it is what the openai-agents scaffolds used to do, so + agents generated before this change carry it. Those scaffolds now clear the + processor list instead, which removes the OpenAI exporter (the thing they were + actually trying to avoid) while leaving spans flowing to the bridge. + + Reads a private attribute, so it is fully guarded: a diagnostic must never be the + reason startup fails, and if upstream renames it we simply stop warning. + """ + try: + from agents.tracing import get_trace_provider + + if getattr(get_trace_provider(), "_disabled", False): + logger.warning( + "The sgp-obs openai-agents bridge is installed but openai-agents " + "tracing is disabled, so Runner turns will produce no model spans. " + "Replace set_tracing_disabled(True) with set_trace_processors([]): " + "that still stops traces reaching api.openai.com, but keeps spans " + "flowing to the bridge." + ) + except Exception: # pragma: no cover - a diagnostic must never break startup + logger.debug("could not determine openai-agents tracing state", exc_info=True) + + +def _warn_if_correlation_backend_mismatched() -> None: + """Warn when sgp-obs is exporting OTel traces but the SDK's business-span + correlation is still reading ddtrace. + + The SDK has had its own correlation for a while (core/tracing/obs_span.py). It + writes BOTH directions of the link between a business span and an obs span: + + forward — obs_trace_id / obs_span_id onto the business span's data, so the + SGP tracing UI can pivot to Tempo + backward — agentex.business_span_id / agentex.business_trace_id onto the OTel + span, so Tempo can pivot back + + Which backend it opens that span in is chosen by SGP_OBS_MODE, which defaults to + ``dd_only``. In that mode it opens a ddtrace span, and only if a ddtrace trace is + already active — which on a bare-uvicorn agent it never is. So the wrapper is + never opened, the correlation dict comes back empty, and BOTH edges vanish + silently while the traces signal still reports itself as wired. + + Measured on sgp-obs 0.16.0 with a real business span: mode unset gives zero + exported spans and no ids in either direction; SGP_OBS_MODE=lgtm gives the + span, both tags, and a round trip that closes (the business span's obs_span_id + equals the exported span's span id, and the span's agentex.business_span_id + equals the business span's id). + + Warn rather than set it: SGP_OBS_MODE also steers correlation reads elsewhere, + and an agent genuinely running ddtrace (the Centipede family) would be misread + if this flipped underneath it. The operator picks; this only makes the silent + case audible. + """ + try: + from agentex.lib.core.tracing.obs_ids import LGTM, get_obs_mode + + if get_obs_mode() != LGTM: + logger.warning( + "sgp-obs wired the traces signal (OpenTelemetry), but SGP_OBS_MODE is " + "%r, so this SDK's business-span correlation still targets ddtrace and " + "will not link anything. Set SGP_OBS_MODE=lgtm to get both edges: " + "obs_trace_id/obs_span_id on the business span, and " + "agentex.business_span_id/agentex.business_trace_id on the OTel span.", + get_obs_mode(), + ) + except Exception: # pragma: no cover - a diagnostic must never break startup + logger.debug("could not check SGP_OBS_MODE", exc_info=True) + + +async def shutdown_sgp_obs() -> None: + """Flush the providers ``init()`` built. Never raises. + + Without this, whatever is sitting in a periodic exporter's buffer when the pod + stops is dropped — which for a short-lived or scaled-to-zero agent can be most + of what it recorded. sgp-obs only flushes providers it OWNS; one adopted from + the runtime is left to its owner, so this is safe under operator injection. + + Run in a thread: the flush blocks up to the SDK export timeout per owned signal, + and this is called from an async lifespan. + """ + if _status is None or not _status.startswith("wired"): + return + + try: + import asyncio + + import sgp_obs # type: ignore[import-not-found] + + # Added in sgp-obs 0.16.0. Feature-detected rather than version-pinned, + # because this package does not depend on sgp-obs and so cannot set a floor. + shutdown = getattr(sgp_obs, "shutdown", None) + if shutdown is None: + logger.debug("sgp-obs has no shutdown(); needs 0.16.0+ to flush on exit") + return + await asyncio.to_thread(shutdown) + except Exception: # pragma: no cover - a failed flush must not fail shutdown + logger.debug("sgp-obs shutdown failed", exc_info=True) + + +def _reset_for_tests() -> None: + global _status + _status = None + # The logging hand-over is a process-wide latch too, and a test that wired the + # logs signal would otherwise leave make_logger attaching nothing for the rest + # of the session. + _logging_reset_for_tests() diff --git a/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py b/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py new file mode 100644 index 000000000..34d2ef3ad --- /dev/null +++ b/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py @@ -0,0 +1,447 @@ +"""Tests for ``agentex.lib.core.observability.sgp_obs_setup``. + +The property under test is that this can never hurt a caller: whatever the state of +sgp-obs or the environment, ``init_sgp_obs`` returns a status string and does not +raise, and ``shutdown_sgp_obs`` does not raise. Both gates get a test, plus the +failure modes, the two silent-misconfiguration warnings, and the flush. + +These never import the real sgp-obs — it is absent in CI by design — so every test +installs a stand-in whose ``init`` is under the test's control. +""" + +from __future__ import annotations + +import sys +import builtins +from contextlib import contextmanager + +import pytest + +from agentex.lib.core.observability import sgp_obs_setup +from agentex.lib.core.observability.sgp_obs_setup import init_sgp_obs, shutdown_sgp_obs + +_SWITCHES = ( + "SGP_OBS_ENABLED", + "SGP_METRICS_DISABLED", + "SGP_TRACES_DISABLED", + "SGP_LOGS_DISABLED", + "AGENT_NAME", +) + + +@pytest.fixture(autouse=True) +def _reset(monkeypatch): + """The status is cached process-wide, so every test starts from unset. The + environment is cleared too: two code paths branch on the master switch, and a + developer with SGP_OBS_ENABLED exported would otherwise flip those tests.""" + for name in _SWITCHES: + monkeypatch.delenv(name, raising=False) + sgp_obs_setup._reset_for_tests() + yield + sgp_obs_setup._reset_for_tests() + + +@contextmanager +def caplog_at(monkeypatch): + """Collect sgp_obs_setup's WARNING messages regardless of root config.""" + records: list[str] = [] + monkeypatch.setattr( + sgp_obs_setup.logger, "warning", + lambda msg, *a, **_k: records.append(msg % a if a else msg), + ) + yield records + + +def _fake_sgp_obs(monkeypatch, init=None, shutdown=None, bridge=None): + """Install a stand-in ``sgp_obs`` module whose entry points we control. + + ``bridge`` stands in for ``sgp_obs.traces.install_openai_agents_bridge``; it lives + on a fake ``sgp_obs.traces`` submodule because that is how the SDK imports it. + """ + module = type(sys)("sgp_obs") + module.init = init if init is not None else (lambda **_kwargs: {"metrics": object()}) + if shutdown is not None: + module.shutdown = shutdown + monkeypatch.setitem(sys.modules, "sgp_obs", module) + + traces = type(sys)("sgp_obs.traces") + traces.install_openai_agents_bridge = bridge if bridge is not None else (lambda: True) + monkeypatch.setitem(sys.modules, "sgp_obs.traces", traces) + return module + + +def _block_sgp_obs_import(monkeypatch, exc=None): + monkeypatch.delitem(sys.modules, "sgp_obs", raising=False) + real_import = builtins.__import__ + error = exc or ImportError("No module named 'sgp_obs'") + + def blocked(name, *args, **kwargs): + if name == "sgp_obs" or name.startswith("sgp_obs."): + raise error + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", blocked) + + +class TestGateOneSgpObsNotInstalled: + def test_missing_package_is_reported_not_raised(self, monkeypatch): + _block_sgp_obs_import(monkeypatch) + assert init_sgp_obs() == "not_installed" + + def test_a_broken_install_does_not_stop_startup(self, monkeypatch): + """An ImportError is ordinary; anything else is a broken install, not a + missing one, and must still be swallowed.""" + _block_sgp_obs_import(monkeypatch, RuntimeError("half-installed wheel")) + assert init_sgp_obs() == "error" + + def test_silence_is_expected_when_nobody_asked(self, monkeypatch, caplog): + """sgp-obs is not a dependency, so absent-and-unasked-for is the normal + case for every agent. It must not warn.""" + _block_sgp_obs_import(monkeypatch) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "not_installed" + assert caplog.records == [] + + def test_enabled_but_missing_says_what_to_install(self, monkeypatch, caplog): + """The one case that must be loud: the operator asked for observability and + the package is not there. Silence would look like working instrumentation.""" + monkeypatch.setenv("SGP_OBS_ENABLED", "true") + _block_sgp_obs_import(monkeypatch) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "not_installed" + assert len(caplog.records) == 1 + assert "sgp-obs is not installed" in caplog.text + assert "genai-auto,http,otlp" in caplog.text + + +class TestGateTwoEnvironmentSwitches: + def test_no_handles_means_disabled(self, monkeypatch): + """sgp_obs.init() returns an empty dict when the master switch or every + per-signal switch is off. That is the DEFAULT: sgp-obs installed, and + recording nothing until someone sets the environment.""" + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {}) + assert init_sgp_obs() == "disabled" + + def test_disabled_and_unasked_for_is_quiet(self, monkeypatch, caplog): + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {}) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "disabled" + assert caplog.records == [] + + def test_master_switch_on_but_nothing_wired_names_the_variables( + self, monkeypatch, caplog + ): + """sgp-obs 0.16.0 made every signal opt-in twice: the master switch plus an + explicit *_DISABLED=false. So SGP_OBS_ENABLED on its own wires nothing and + says nothing, which is the single easiest way to believe an agent is + instrumented when it is not.""" + monkeypatch.setenv("SGP_OBS_ENABLED", "true") + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {}) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "disabled" + assert len(caplog.records) == 1 + for var in ("SGP_METRICS_DISABLED", "SGP_TRACES_DISABLED", "SGP_LOGS_DISABLED"): + assert var in caplog.text + + @pytest.mark.parametrize("raw", ["1", "true", "TRUE", "yes", "on"]) + def test_master_switch_truthy_forms(self, monkeypatch, caplog, raw): + """Matched to sgp_obs.env._TRUTHY, so this module's idea of "on" is the + same as the library's. A mismatch would put the warning on the wrong side.""" + monkeypatch.setenv("SGP_OBS_ENABLED", raw) + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {}) + with caplog.at_level("WARNING"): + init_sgp_obs() + assert len(caplog.records) == 1 + + def test_traces_without_lgtm_mode_warns_that_correlation_is_dead( + self, monkeypatch + ): + """SGP_OBS_MODE defaults to dd_only, where the SDK's business-span wrapper + only opens if a ddtrace trace is already active — never true on a + bare-uvicorn agent. So both correlation edges vanish while the traces + signal still reports itself wired. Measured: mode unset -> zero exported + spans and no ids either way; lgtm -> both edges, round trip closes.""" + monkeypatch.delenv("SGP_OBS_MODE", raising=False) + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {"traces": object()}) + with caplog_at(monkeypatch) as records: + assert init_sgp_obs() == "wired:traces" + assert any("SGP_OBS_MODE" in r for r in records) + + def test_traces_with_lgtm_mode_is_quiet(self, monkeypatch, caplog): + monkeypatch.setenv("SGP_OBS_MODE", "lgtm") + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {"traces": object()}) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "wired:traces" + assert caplog.records == [] + + def test_metrics_only_does_not_warn_about_the_mode(self, monkeypatch, caplog): + """The correlation edges are a traces concern. A metrics-only agent has no + business-span linking to lose, so the warning would be noise.""" + monkeypatch.delenv("SGP_OBS_MODE", raising=False) + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {"metrics": object()}) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "wired:metrics" + assert caplog.records == [] + + def test_all_three_signals_are_named_in_the_status(self, monkeypatch): + _fake_sgp_obs( + monkeypatch, + lambda **_kwargs: {"logs": object(), "metrics": object(), "traces": object()}, + ) + assert init_sgp_obs() == "wired:logs,metrics,traces" + + +class TestWhatIsPassedToSgpObs: + @staticmethod + def _capture(monkeypatch): + seen = {} + + def capture(**kwargs): + seen.update(kwargs) + return {"metrics": object()} + + _fake_sgp_obs(monkeypatch, capture) + return seen + + def test_app_reaches_sgp_obs(self, monkeypatch): + """Passing the ACP server is what adds http.server.* for the agent's own + entry point and installs the trace-context ingress, so it must not be + silently dropped.""" + seen = self._capture(monkeypatch) + sentinel = object() + init_sgp_obs(app=sentinel) + assert seen["app"] is sentinel + + def test_source_is_agentex(self, monkeypatch): + """The SDK knows the runtime; an agent author would have to know to pass it. + It is what stamps agent_id and task_id onto log records.""" + seen = self._capture(monkeypatch) + init_sgp_obs() + assert seen["source"] == "agentex" + + def test_agent_name_is_offered_as_the_service_name(self, monkeypatch): + """sgp-obs fills OTEL_SERVICE_NAME from this only when the deployment left + it unset; without either, every signal is attributed to "unknown".""" + monkeypatch.setenv("AGENT_NAME", "compass-sleep-agent") + seen = self._capture(monkeypatch) + init_sgp_obs() + assert seen["service_name"] == "compass-sleep-agent" + + @pytest.mark.parametrize("raw", ["", " "]) + def test_blank_agent_name_is_passed_as_none(self, monkeypatch, raw): + """Blank is the Helm rendered-empty idiom. Forwarding "" would have sgp-obs + set OTEL_SERVICE_NAME to an empty string rather than leave it alone.""" + monkeypatch.setenv("AGENT_NAME", raw) + seen = self._capture(monkeypatch) + init_sgp_obs() + assert seen["service_name"] is None + + +class TestFailOpen: + def test_an_exception_from_init_is_swallowed(self, monkeypatch): + def boom(**_kwargs): + raise ValueError("boom") + + _fake_sgp_obs(monkeypatch, boom) + assert init_sgp_obs() == "error" + + def test_a_ci_logs_misconfiguration_still_does_not_stop_startup(self, monkeypatch): + """sgp_obs.init has one deliberate exception to its own fail-open rule: under + the CI variable, a logs misconfiguration raises. An agent must still serve.""" + + def strict(**_kwargs): + raise RuntimeError("MisconfigurationError: drop mode without an allowlist") + + _fake_sgp_obs(monkeypatch, strict) + assert init_sgp_obs() == "error" + + def test_status_is_computed_once(self, monkeypatch): + """A Temporal worker and an ACP server can both reach this in one process; + sgp_obs.init() is not meant to run twice.""" + calls = [] + + def counting(**kwargs): + calls.append(kwargs) + return {"metrics": object()} + + _fake_sgp_obs(monkeypatch, counting) + assert init_sgp_obs() == "wired:metrics" + assert init_sgp_obs() == "wired:metrics" + assert len(calls) == 1 + + +class TestShutdown: + async def test_flushes_when_wired(self, monkeypatch): + """Without this the periodic exporter's buffer is dropped when the pod + stops, which for a short-lived agent can be most of what it recorded.""" + called = [] + _fake_sgp_obs(monkeypatch, shutdown=lambda: called.append(True)) + assert init_sgp_obs() == "wired:metrics" + await shutdown_sgp_obs() + assert called == [True] + + async def test_no_flush_when_never_wired(self, monkeypatch): + called = [] + _fake_sgp_obs( + monkeypatch, init=lambda **_kwargs: {}, shutdown=lambda: called.append(True) + ) + assert init_sgp_obs() == "disabled" + await shutdown_sgp_obs() + assert called == [] + + async def test_no_flush_before_init(self, monkeypatch): + """Called from the lifespan's finally, which runs even if startup failed + before the constructor's init_sgp_obs ever ran.""" + called = [] + _fake_sgp_obs(monkeypatch, shutdown=lambda: called.append(True)) + await shutdown_sgp_obs() + assert called == [] + + async def test_an_older_sgp_obs_without_shutdown_is_tolerated(self, monkeypatch): + """shutdown() arrived in 0.16.0. This package declares no dependency on + sgp-obs and so cannot set a floor, hence feature detection.""" + _fake_sgp_obs(monkeypatch) # no shutdown attribute + assert init_sgp_obs() == "wired:metrics" + await shutdown_sgp_obs() # must not raise + + async def test_a_failing_flush_does_not_fail_shutdown(self, monkeypatch): + def boom(): + raise RuntimeError("exporter timed out") + + _fake_sgp_obs(monkeypatch, shutdown=boom) + assert init_sgp_obs() == "wired:metrics" + await shutdown_sgp_obs() # must not raise + + +class TestAnAgentStillServesWithoutSgpObs: + """Nitesh's verification item, startup half: an account not yet on the + CodeArtifact allowlist gets an image with no ``sgp_obs`` in it. The gate + returning ``not_installed`` is necessary but not sufficient — what has to hold + is that the ACP server still constructs and still answers requests. This + exercises the real constructor, which is where ``init_sgp_obs`` is called. + """ + + def test_acp_server_constructs_and_serves_healthz(self, monkeypatch): + from fastapi.testclient import TestClient + + from agentex.lib.sdk.fastacp.base.base_acp_server import BaseACPServer + + # Import first, unpatched, so the deep FastACP dependency chain loads + # cleanly; only sgp_obs is hidden, and only while the constructor runs. + _block_sgp_obs_import(monkeypatch) + + server = BaseACPServer() + assert sgp_obs_setup._status == "not_installed" + + # No `with`: that would run the lifespan, which registers the agent + # against a live control plane. + response = TestClient(server).get("/healthz") + assert response.status_code == 200 + assert response.json() == {"status": "healthy"} + + def test_the_json_rpc_route_is_still_mounted(self, monkeypatch): + """A server that answers /healthz but lost /api would pass a liveness probe + and fail every actual request.""" + from agentex.lib.sdk.fastacp.base.base_acp_server import BaseACPServer + + _block_sgp_obs_import(monkeypatch) + routes = {getattr(r, "path", None) for r in BaseACPServer().routes} + assert {"/healthz", "/api"} <= routes + + +class TestOpenAIAgentsBridge: + """sgp_obs.init() installs the GenAI attempt processor, the litellm adapter and the + egress instrumentors by itself, but NOT the openai-agents bridge (measured on + 0.16.0). That is the path ~83% of model-calling agents take, so the SDK installs it + — otherwise "traces on" produces no logical model-operation spans for most agents. + """ + + def test_installed_when_traces_are_wired(self, monkeypatch): + calls = [] + _fake_sgp_obs( + monkeypatch, + init=lambda **_kwargs: {"traces": object()}, + bridge=lambda: calls.append(True) or True, + ) + monkeypatch.setenv("SGP_OBS_MODE", "lgtm") + assert init_sgp_obs() == "wired:traces" + assert calls == [True] + + def test_not_installed_without_the_traces_signal(self, monkeypatch): + """A metrics-only agent has no span pipeline to feed, so installing an + openai-agents trace processor would be pointless work at startup.""" + calls = [] + _fake_sgp_obs( + monkeypatch, + init=lambda **_kwargs: {"metrics": object()}, + bridge=lambda: calls.append(True) or True, + ) + assert init_sgp_obs() == "wired:metrics" + assert calls == [] + + def test_a_bridge_that_declines_is_reported(self, monkeypatch, caplog): + """False means the `agents` SDK was not importable. openai-agents is a hard + dependency of this package, so that should be impossible — say so rather than + swallow it.""" + monkeypatch.setenv("SGP_OBS_MODE", "lgtm") + _fake_sgp_obs( + monkeypatch, init=lambda **_kwargs: {"traces": object()}, bridge=lambda: False + ) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "wired:traces" + assert "openai-agents bridge" in caplog.text + + def test_a_raising_bridge_does_not_stop_startup(self, monkeypatch): + def boom(): + raise RuntimeError("sgp-obs internals moved") + + monkeypatch.setenv("SGP_OBS_MODE", "lgtm") + _fake_sgp_obs( + monkeypatch, init=lambda **_kwargs: {"traces": object()}, bridge=boom + ) + assert init_sgp_obs() == "wired:traces" + + +class TestLoggingHandover: + """agentex's make_logger attaches a handler to each module's own logger; sgp-obs' + logs pipeline owns the ROOT logger and deliberately leaves named loggers alone. Both + then print, so every record appears twice — and the agentex copy is emitted before + the pipeline's filters, so it carries no agent_id/task_id, is not governed by the + allowlist, and is not truncated. + """ + + @staticmethod + def _spy(monkeypatch): + calls = [] + monkeypatch.setattr( + sgp_obs_setup, "route_agentex_loggers_to_root", lambda: calls.append(True) or 1 + ) + return calls + + def test_handover_runs_when_the_logs_signal_is_wired(self, monkeypatch): + calls = self._spy(monkeypatch) + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {"logs": object()}) + assert init_sgp_obs() == "wired:logs" + assert calls == [True] + + def test_no_handover_when_logs_are_not_wired(self, monkeypatch): + """Nothing owns the root logger in that case, so stripping the leaf handlers + would send agentex's records nowhere at all.""" + calls = self._spy(monkeypatch) + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {"metrics": object()}) + assert init_sgp_obs() == "wired:metrics" + assert calls == [] + + def test_no_handover_when_sgp_obs_is_absent(self, monkeypatch): + calls = self._spy(monkeypatch) + _block_sgp_obs_import(monkeypatch) + assert init_sgp_obs() == "not_installed" + assert calls == [] + + def test_a_failing_handover_does_not_stop_startup(self, monkeypatch): + def boom(): + raise RuntimeError("logging registry is in a strange state") + + monkeypatch.setattr(sgp_obs_setup, "route_agentex_loggers_to_root", boom) + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {"logs": object()}) + assert init_sgp_obs() == "wired:logs" diff --git a/src/agentex/lib/core/temporal/workers/worker.py b/src/agentex/lib/core/temporal/workers/worker.py index 72631917f..651286fbe 100644 --- a/src/agentex/lib/core/temporal/workers/worker.py +++ b/src/agentex/lib/core/temporal/workers/worker.py @@ -32,6 +32,8 @@ from agentex.lib.core.tracing.temporal import temporal_tracing_interceptors from agentex.lib.environment_variables import EnvironmentVariables from agentex.lib.core.compat.version_guard import assert_backend_compatible +from agentex.lib.core.observability.sgp_obs_setup import init_sgp_obs, shutdown_sgp_obs +from agentex.lib.core.tracing.tracing_processor_manager import shutdown_sync_tracing_processors logger = make_logger(__name__) @@ -222,6 +224,18 @@ async def run( workflow: type | None = None, workflows: list[type] | None = None, ): + # A Temporal agent runs its model calls HERE, in a separate process from the + # ACP server, and this process never constructs a BaseACPServer — so without + # this call an agent that installed sgp-obs and set the documented environment + # would still get no metrics, traces or structured logs from its worker, which + # is where the interesting work happens. + # + # No `app=`: there is no ASGI application in this process. The health-check + # server is aiohttp, which sgp-obs' ASGI middleware does not apply to, so the + # worker contributes model and egress telemetry but no http.server.* — correct, + # since nothing here serves agent traffic. + init_sgp_obs() + await self.start_health_check_server() await self._register_agent() @@ -267,7 +281,14 @@ async def run( # Eagerly set the worker status to healthy self.healthy = True logger.info(f"Running workers for task queue: {self.task_queue}") - await worker.run() + try: + await worker.run() + finally: + # Same drains as the ACP lifespan, for the same reason: whatever is still + # queued when the pod stops is otherwise dropped. Both are bounded and + # fail-open, so neither can stop the worker exiting. + await shutdown_sync_tracing_processors() + await shutdown_sgp_obs() async def _health_check(self): return web.json_response(self.healthy) diff --git a/src/agentex/lib/core/tracing/tracing_processor_manager.py b/src/agentex/lib/core/tracing/tracing_processor_manager.py index 07c440313..5227e891c 100644 --- a/src/agentex/lib/core/tracing/tracing_processor_manager.py +++ b/src/agentex/lib/core/tracing/tracing_processor_manager.py @@ -1,5 +1,8 @@ from __future__ import annotations +import asyncio +import logging +import threading from typing import TYPE_CHECKING from threading import Lock @@ -78,3 +81,104 @@ def get_sync_tracing_processors(): def get_async_tracing_processors(): return GLOBAL_TRACING_PROCESSOR_MANAGER.get_async_processors() + + +_logger = logging.getLogger(__name__) + +# Total wall-clock budget for draining every sync tracing processor. A pod's +# terminationGracePeriodSeconds (30s by default) is shared with the OTel flush that +# follows this, so the drain takes a small slice of it. +SYNC_TRACING_SHUTDOWN_BUDGET_S = 5.0 + + +async def shutdown_sync_tracing_processors( + budget_s: float = SYNC_TRACING_SHUTDOWN_BUDGET_S, +) -> None: + """Drain the sync tracing processors' queues at shutdown. Never raises. + + Nothing used to call this. The ACP lifespan drained ``shutdown_default_span_queue``, + which is the ASYNC path only, so a sync agent dropped whatever business spans were + still queued when the pod stopped. That matters beyond the lost spans: the business + span is what an obs span's ``agentex.business_trace_id`` resolves to, so losing it + breaks the pivot from Tempo back to the SGP store. + + ``SGPSyncTracingProcessor.shutdown`` calls ``flush_queue()``, a BLOCKING HTTP flush + with retries, so three properties have to hold at once: + + **Off the calling loop.** Awaiting it inline stalls the lifespan, so a slow + collector could burn the pod's whole termination grace period and stop the OTel + flush that runs after this — trading a few business spans for all of the OTel ones. + + **Concurrent.** Every processor is started at once and they share one deadline. A + sequential loop would let the first stalled processor spend the entire budget, so + later processors were skipped even when they would have finished instantly. + + **On DAEMON threads, not the default executor.** This is the subtle one. + ``asyncio.wait_for`` stops *awaiting* a thread; it cannot stop the thread. And + ``asyncio.run`` calls ``loop.shutdown_default_executor()``, which JOINS the default + executor — as does a private ``ThreadPoolExecutor``, via its atexit hook. So a + timed-out ``asyncio.to_thread`` flush leaves the process blocked on the very export + the deadline was meant to escape. Measured: a 10s stalled flush under a 0.25s budget + returns in 0.25s but the process exits at 10.0s with ``to_thread``, and at 0.25s on + a daemon thread. A daemon thread is abandoned at interpreter exit, which is what the + budget promises. + """ + try: + processors = get_sync_tracing_processors() + except Exception: # pragma: no cover - nothing to drain + _logger.debug("sync tracing processors unavailable at shutdown", exc_info=True) + return + + if not processors: + return + + loop = asyncio.get_running_loop() + finished: list[threading.Event] = [] + all_done = asyncio.Event() + + def _note_finished() -> None: + if all(event.is_set() for event in finished): + all_done.set() + + def _flush(processor: SyncTracingProcessor, event: threading.Event) -> None: + try: + processor.shutdown() + except Exception: + _logger.warning( + "%s raised while flushing on shutdown; some business spans may be lost", + type(processor).__name__, + exc_info=True, + ) + finally: + event.set() + # The loop may already be closed if we timed out and shutdown raced ahead; + # abandoning the notification is fine, nobody is waiting on it any more. + try: + loop.call_soon_threadsafe(_note_finished) + except RuntimeError: # pragma: no cover - loop already closed + pass + + for index, processor in enumerate(processors): + event = threading.Event() + finished.append(event) + threading.Thread( + target=_flush, + args=(processor, event), + daemon=True, + name=f"agentex-span-flush-{index}", + ).start() + + try: + await asyncio.wait_for(all_done.wait(), budget_s) + except (TimeoutError, asyncio.TimeoutError): + stalled = [ + type(processor).__name__ + for processor, event in zip(processors, finished) + if not event.is_set() + ] + _logger.warning( + "sync tracing shutdown budget of %.1fs expired with %s still flushing; " + "their business spans are lost, but shutdown continues", + budget_s, + ", ".join(stalled) or "unknown processors", + ) diff --git a/src/agentex/lib/sdk/fastacp/base/base_acp_server.py b/src/agentex/lib/sdk/fastacp/base/base_acp_server.py index 864b466d0..06bc2595c 100644 --- a/src/agentex/lib/sdk/fastacp/base/base_acp_server.py +++ b/src/agentex/lib/sdk/fastacp/base/base_acp_server.py @@ -39,6 +39,8 @@ FASTACP_HEADER_SKIP_EXACT, FASTACP_HEADER_SKIP_PREFIXES, ) +from agentex.lib.core.observability.sgp_obs_setup import init_sgp_obs, shutdown_sgp_obs +from agentex.lib.core.tracing.tracing_processor_manager import shutdown_sync_tracing_processors logger = make_logger(__name__) @@ -139,6 +141,20 @@ def __init__(self): # Method handlers # this just adds a request ID to the request and response headers self.add_middleware(RequestIDMiddleware) + + # Optional observability (traces, metrics, logs), off unless sgp-obs is + # installed AND the SGP_OBS_* environment switches ask for it — see + # observability/sgp_obs_setup.py for the two gates. sgp-obs is deliberately + # not a dependency of this package; the agent declares it. Returns a status + # instead of raising: a telemetry problem must never stop an agent starting. + # + # Here rather than in the lifespan, deliberately: sgp-obs installs ASGI + # instrumentation via add_middleware, and Starlette raises "Cannot add middleware + # after an application has started" once the lifespan is running. Wiring it there + # loses http.server.* for the agent's own entry point — and loses it QUIETLY, + # because sgp-obs fails open. + init_sgp_obs(app=self) + self._handlers: dict[RPCMethod, Callable] = {} # Agent info to return in healthz @@ -176,9 +192,20 @@ async def lifespan_context(app: FastAPI): # noqa: ARG001 yield finally: await shutdown_default_span_queue() + # The queue above is the ASYNC path only. Sync tracing processors + # hold their own queue and nothing ever drained it, so a sync ACP + # agent lost whatever business spans were still queued when the pod + # stopped — including the ones the obs correlation points at. + await shutdown_sync_tracing_processors() + # Flush whatever sgp-obs still holds. A periodic exporter's buffer + # is otherwise dropped when the pod stops, which for a short-lived + # or scaled-to-zero agent can be most of what it recorded. No-op + # when sgp-obs is absent or was never wired. + await shutdown_sgp_obs() return lifespan_context + async def _healthz(self): """Health check endpoint""" result = {"status": "healthy"} diff --git a/src/agentex/lib/sdk/fastacp/base/tests/__init__.py b/src/agentex/lib/sdk/fastacp/base/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py b/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py new file mode 100644 index 000000000..fe9cfbe28 --- /dev/null +++ b/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py @@ -0,0 +1,302 @@ +"""Tests for the ACP lifespan's shutdown drains. + +``shutdown_default_span_queue`` covers the async span path. The SYNC tracing +processors keep their own queue, and nothing in the SDK ever shut them down, so a +sync ACP agent dropped whatever business spans were still queued when the pod +stopped. That is worse than the spans themselves: the business span is what an obs +span's ``agentex.business_trace_id`` resolves to, so losing it breaks the pivot from +Tempo back to the SGP store. +""" + +from __future__ import annotations + +from agentex.lib.sdk.fastacp.base import base_acp_server +from agentex.lib.core.tracing.tracing_processor_manager import ( + shutdown_sync_tracing_processors, +) + + +def _block_sgp_obs_import(monkeypatch): + """Make `import sgp_obs` fail, i.e. the image a tokenless build produces.""" + import sys + import builtins + + monkeypatch.delitem(sys.modules, "sgp_obs", raising=False) + real_import = builtins.__import__ + + def blocked(name, *args, **kwargs): + if name == "sgp_obs" or name.startswith("sgp_obs."): + raise ImportError("No module named 'sgp_obs'") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", blocked) + + +class _Processor: + def __init__(self, explode: bool = False) -> None: + self.calls = 0 + self._explode = explode + + def shutdown(self) -> None: + self.calls += 1 + if self._explode: + raise RuntimeError("flush timed out") + + +def _patch_processors(monkeypatch, processors): + import agentex.lib.core.tracing.tracing_processor_manager as mgr + + monkeypatch.setattr(mgr, "get_sync_tracing_processors", lambda: processors) + + +class TestSyncProcessorDrain: + async def test_every_processor_is_flushed(self, monkeypatch): + a, b = _Processor(), _Processor() + _patch_processors(monkeypatch, [a, b]) + await shutdown_sync_tracing_processors() + assert (a.calls, b.calls) == (1, 1) + + async def test_one_failure_does_not_stop_the_others(self, monkeypatch): + """A processor that hangs or raises must not strand the spans held by the + ones after it in the list.""" + bad, good = _Processor(explode=True), _Processor() + _patch_processors(monkeypatch, [bad, good]) + await shutdown_sync_tracing_processors() + assert good.calls == 1 + + async def test_no_processors_is_a_no_op(self, monkeypatch): + _patch_processors(monkeypatch, []) + await shutdown_sync_tracing_processors() # must not raise + + async def test_an_unreadable_processor_list_does_not_fail_shutdown(self, monkeypatch): + """Nothing here may stop the pod from shutting down. + + This used to block the import of ``tracing_processor_manager``, which tested + nothing once the drain moved INTO that module: it reads + ``get_sync_tracing_processors`` as a module global, so the import never runs and + the ``except`` branch was never reached. Make the lookup itself raise instead.""" + import agentex.lib.core.tracing.tracing_processor_manager as mgr + + def boom(): + raise RuntimeError("processor registry unavailable") + + monkeypatch.setattr(mgr, "get_sync_tracing_processors", boom) + await shutdown_sync_tracing_processors() # must not raise + + def test_the_lifespan_calls_it(self): + """Pin the wiring, not just the helper: a drain nothing calls is worthless.""" + import inspect + + source = inspect.getsource(base_acp_server.BaseACPServer.get_lifespan_function) + assert "shutdown_sync_tracing_processors()" in source + assert "shutdown_sgp_obs()" in source + + +class TestTheDrainIsBounded: + """`SGPSyncTracingProcessor.shutdown` does a BLOCKING HTTP flush with retries. If + the drain waited on it inline and without a limit, a slow or unreachable collector + would burn the pod's whole termination grace period and the OTel flush that runs + after it would never happen — trading a few business spans for all of the OTel ones. + """ + + async def test_a_stalled_processor_does_not_hang_shutdown(self, monkeypatch): + import time + import asyncio + + class Stalled: + def shutdown(self): + time.sleep(2) # blocking, like a retrying HTTP flush + + _patch_processors(monkeypatch, [Stalled()]) + started = asyncio.get_running_loop().time() + await shutdown_sync_tracing_processors(budget_s=0.25) + elapsed = asyncio.get_running_loop().time() - started + assert elapsed < 1, f"drain took {elapsed:.1f}s against a 0.25s budget" + + async def test_the_budget_is_shared_so_a_stall_cannot_starve_the_rest(self, monkeypatch): + """A shared deadline means the drain as a whole is bounded, not each processor + separately — N stalled processors must not cost N * budget.""" + import time + import asyncio + + class Stalled: + def shutdown(self): + time.sleep(2) + + _patch_processors(monkeypatch, [Stalled(), Stalled(), Stalled()]) + started = asyncio.get_running_loop().time() + await shutdown_sync_tracing_processors(budget_s=0.25) + elapsed = asyncio.get_running_loop().time() - started + assert elapsed < 1, f"drain took {elapsed:.1f}s for 3 stalled processors" + + async def test_it_does_not_block_the_event_loop(self, monkeypatch): + """The flush must run off-loop: other lifespan work has to keep progressing + while a processor is stuck.""" + import time + import asyncio + + class Stalled: + def shutdown(self): + time.sleep(2) + + _patch_processors(monkeypatch, [Stalled()]) + ticks = 0 + + async def heartbeat(): + nonlocal ticks + while True: + await asyncio.sleep(0.01) + ticks += 1 + + beat = asyncio.create_task(heartbeat()) + await shutdown_sync_tracing_processors(budget_s=0.25) + beat.cancel() + assert ticks > 0, "the event loop was blocked during the drain" + + +class TestTheTemporalWorkerIsWiredToo: + """A Temporal agent runs its model calls in the worker process, which never + constructs a BaseACPServer. Without its own init the documented environment leaves + that process — the one doing the interesting work — completely unwired. + """ + + def test_the_worker_inits_and_drains(self): + import inspect + + from agentex.lib.core.temporal.workers.worker import AgentexWorker + + source = inspect.getsource(AgentexWorker.run) + assert "init_sgp_obs()" in source + assert "shutdown_sgp_obs()" in source + assert "shutdown_sync_tracing_processors()" in source + + def test_the_worker_does_not_pass_an_app(self): + """There is no ASGI application in the worker process. The health-check server + is aiohttp, which sgp-obs' ASGI middleware does not apply to, so passing it + would be wrong rather than merely useless.""" + import inspect + + from agentex.lib.core.temporal.workers.worker import AgentexWorker + + source = inspect.getsource(AgentexWorker.run) + assert "init_sgp_obs(app=" not in source + + +class TestConcurrencyAndProcessExit: + """Two properties the budget only really has if these hold.""" + + async def test_a_fast_processor_finishes_even_when_another_stalls(self, monkeypatch): + """Flushes start concurrently under ONE shared deadline. Draining them in + sequence let the first stalled processor spend the whole budget, so every + processor after it was skipped even when it would have returned instantly.""" + import time + + class Stalled: + def shutdown(self): + time.sleep(2) + + class Fast: + def __init__(self): + self.flushed = False + + def shutdown(self): + self.flushed = True + + fast = Fast() + # Stalled FIRST: in a sequential drain it would eat the budget and `fast` + # would never be asked. + _patch_processors(monkeypatch, [Stalled(), fast]) + await shutdown_sync_tracing_processors(budget_s=0.5) + assert fast.flushed, "a fast processor was starved by a stalled one" + + def test_a_stalled_flush_does_not_delay_process_exit(self): + """The property the deadline actually promises, and the one it did NOT have. + + `asyncio.wait_for` stops awaiting a thread; it cannot stop the thread. And + `asyncio.run` joins the default executor on the way out (as does a private + ThreadPoolExecutor, via its atexit hook), so a timed-out `asyncio.to_thread` + flush left the process blocked on the very export the budget was meant to + escape — measured at 10.0s against a 0.25s budget. Daemon threads are abandoned + at interpreter exit, which is what the budget promises. + + A subprocess, because this is about interpreter shutdown: it cannot be observed + from inside the test process. + """ + import os + import sys + import time + import textwrap + import subprocess + from pathlib import Path + + # tests/base/fastacp/sdk/lib/agentex/src -> parents[6] is the src root. + src = Path(__file__).resolve().parents[6] + program = textwrap.dedent( + """ + import asyncio, sys, time + from agentex.lib.core.tracing.tracing_processor_manager import ( + shutdown_sync_tracing_processors, + ) + import agentex.lib.core.tracing.tracing_processor_manager as mgr + + class Stalled: + def shutdown(self): + time.sleep(30) + + mgr.get_sync_tracing_processors = lambda: [Stalled()] + asyncio.run(shutdown_sync_tracing_processors(budget_s=0.25)) + """ + ) + started = time.monotonic() + proc = subprocess.run( + [sys.executable, "-c", program], + capture_output=True, + text=True, + timeout=30, + # Inherit the environment: replacing it wholesale breaks the + # interpreter's own bootstrap before the test can run. + env={**os.environ, "PYTHONPATH": str(src)}, + ) + elapsed = time.monotonic() - started + assert proc.returncode == 0, proc.stderr[-2000:] + assert elapsed < 10, ( + f"process took {elapsed:.1f}s to exit with a 30s stalled flush and a " + "0.25s budget; the flush thread is blocking interpreter shutdown" + ) + + +class TestTheWorkerObsPathRunsWithoutSgpObs: + """The image a build with NO broker token produces has no sgp-obs in it, and a + Temporal agent's model calls happen in this process. + + The two tests above pin that ``run()`` *calls* these, by reading its source. That + cannot catch a call that is written correctly and then raises, so this exercises the + sequence for real. Together: one proves the wiring exists, the other proves it is + harmless. + """ + + def test_the_worker_module_imports_and_constructs(self): + from agentex.lib.core.temporal.workers.worker import AgentexWorker + + # port 0 so nothing binds a real health port during the test + assert AgentexWorker(task_queue="probe", health_check_port=0) is not None + + async def test_init_and_both_drains_are_inert(self, monkeypatch): + """Exactly what ``run()`` does: init at entry, both drains in its finally — + with nothing wired, which is every agent that has not adopted.""" + from agentex.lib.core.observability import sgp_obs_setup + from agentex.lib.core.observability.sgp_obs_setup import ( + init_sgp_obs, + shutdown_sgp_obs, + ) + + monkeypatch.delenv("SGP_OBS_ENABLED", raising=False) + sgp_obs_setup._reset_for_tests() + try: + _block_sgp_obs_import(monkeypatch) + assert init_sgp_obs() == "not_installed" + # Neither drain may raise just because nothing was ever wired. + await shutdown_sync_tracing_processors() + await shutdown_sgp_obs() + finally: + sgp_obs_setup._reset_for_tests() diff --git a/src/agentex/lib/utils/logging.py b/src/agentex/lib/utils/logging.py index a0d39331b..fd97fc072 100644 --- a/src/agentex/lib/utils/logging.py +++ b/src/agentex/lib/utils/logging.py @@ -13,6 +13,25 @@ DEFAULT_LOG_LEVEL = logging.INFO +# Every logger this module hands out is a LEAF (``make_logger(__name__)``), and until +# now each one carried its own handler. That is fine on its own, but an observability +# pipeline that owns the ROOT logger -- sgp-obs replaces the root handler list -- then +# prints a SECOND copy of every record: once here, and once more when the record +# propagates to root. Measured on sgp-obs 0.16.0: one ``logger.info()`` produced two +# stdout lines, and sgp-obs' own boot warning named 63 loggers "bypassing log +# governance". The plain-text copy also skips the pipeline's enrichment (agent_id, +# task_id), its allowlist and its truncation, so it is not merely redundant. +# +# While this is True, ``make_logger`` attaches nothing and the record reaches the root +# pipeline by propagation alone. ``sgp_obs_setup`` sets it via +# :func:`route_agentex_loggers_to_root` -- nothing else may. +_ROOT_PIPELINE_OWNS_LOGGING = False + +# Handlers are cleared by prefix rather than by an enumerated list: the names are +# module paths, several agentex modules are imported LAZILY, and any list would be a +# snapshot that goes stale the moment one of them loads. +_PACKAGE_ROOT = "agentex" + def resolve_log_level() -> int: """Read the log level from ``LOG_LEVEL``, falling back to INFO. @@ -72,6 +91,13 @@ def make_logger(name: str) -> logging.Logger: logger = logging.getLogger(name) logger.setLevel(resolve_log_level()) + if _ROOT_PIPELINE_OWNS_LOGGING: + # A handler here would be the second one on this record's path to stdout. + # The level above is deliberately still applied: LOG_LEVEL is what agent + # authors set, and letting the pipeline's own threshold silently replace it + # would change behaviour nobody asked to change. + return logger + environment = os.getenv("ENVIRONMENT") if environment == "local": console = Console() @@ -96,3 +122,60 @@ def make_logger(name: str) -> logging.Logger: logger.addHandler(stream_handler) # Create a logger object with the name of the current module return logger + + +def route_agentex_loggers_to_root() -> int: + """Hand agentex's logging over to whatever owns the root logger. Returns the + number of loggers cleared. + + Two halves, and BOTH are needed -- measured, one line per ``logger.info()`` only + when they run together: + + * the sweep below fixes the loggers that ALREADY exist, i.e. every agentex module + imported before this ran; + * the flag fixes every logger created AFTER it, which a sweep cannot reach. + agentex imports several modules lazily (the adk ``_claude_code_sync`` / + ``_codex_sync`` / ``_pydantic_ai_sync`` harnesses among them), so their + ``make_logger`` call happens later and would attach a fresh duplicate handler. + + sgp-obs offers ``capture_loggers=`` for the first half, and it is deliberately not + used: it matches EXACT logger names, not prefixes (measured -- passing + ``("agentex",)`` still produced two lines), so it would mean enumerating ~60 module + paths; and passing anything at all replaces its uvicorn default, which would put + uvicorn's access log back to printing twice. + + Only agentex's own loggers are touched. A third party's handler may be there on + purpose -- which is exactly why sgp-obs warns about them rather than stripping them + -- so litellm's three loggers and anything else keep whatever they have. + """ + global _ROOT_PIPELINE_OWNS_LOGGING + _ROOT_PIPELINE_OWNS_LOGGING = True + + cleared = 0 + # list() snapshots the registry: a getLogger() on another thread would otherwise + # mutate the dict mid-iteration. + for name, existing in list(logging.Logger.manager.loggerDict.items()): + if not isinstance(existing, logging.Logger): + continue # a PlaceHolder for a name whose children exist but itself does not + if name != _PACKAGE_ROOT and not name.startswith(_PACKAGE_ROOT + "."): + continue + if not existing.handlers: + continue + if not existing.propagate: + # Deliberately cut off from root, so nothing of its reaches the pipeline. + # Clearing its handlers would send its records NOWHERE -- worse than a + # duplicate. Leave it exactly as its owner set it up. + continue + for handler in list(existing.handlers): + try: + handler.flush() # a buffering handler must not lose records on removal + except Exception: + pass + existing.removeHandler(handler) + cleared += 1 + return cleared + + +def _reset_for_tests() -> None: + global _ROOT_PIPELINE_OWNS_LOGGING + _ROOT_PIPELINE_OWNS_LOGGING = False diff --git a/src/agentex/lib/utils/tests/__init__.py b/src/agentex/lib/utils/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/agentex/lib/utils/tests/test_logging_handover.py b/src/agentex/lib/utils/tests/test_logging_handover.py new file mode 100644 index 000000000..4a7205afb --- /dev/null +++ b/src/agentex/lib/utils/tests/test_logging_handover.py @@ -0,0 +1,182 @@ +"""Tests for handing agentex's loggers over to a root logging pipeline. + +``make_logger`` attaches a handler to each module's OWN (leaf) logger. sgp-obs' logs +pipeline replaces the handlers on the ROOT logger and deliberately leaves named loggers +alone, on the grounds that a named logger's handler may be there on purpose. Each is +defensible; together they print every record twice — once in agentex's plain text from +the leaf, once as pipeline JSON from root. Measured on sgp-obs 0.16.0: one +``logger.info()`` produced two stdout lines and sgp-obs named 63 loggers as "bypassing +log governance". + +The duplicate is not merely redundant. It is emitted before the pipeline's filters, so +it carries no ``agent_id``/``task_id``, is not governed by the allowlist, and is not +truncated. + +The fix has two halves and needs both, which is what the subprocess tests pin: + +* the sweep clears loggers that ALREADY exist when it runs; +* the latch stops ``make_logger`` attaching to loggers created AFTERWARDS. + +A sweep alone misses the second: agentex imports several harness modules lazily, so +their ``make_logger`` runs later and would attach a fresh duplicate. +""" + +from __future__ import annotations + +import os +import sys +import logging +import textwrap +import subprocess +from typing import override +from pathlib import Path + +import pytest + +from agentex.lib.utils import logging as agentex_logging +from agentex.lib.utils.logging import make_logger, route_agentex_loggers_to_root + +_SRC = Path(__file__).resolve().parents[4] + + +@pytest.fixture(autouse=True) +def _restore_logging(): + """The latch and the loggers are process-wide; put both back.""" + saved = { + name: (obj.handlers[:], obj.propagate) + for name, obj in logging.Logger.manager.loggerDict.items() + if isinstance(obj, logging.Logger) + } + try: + yield + finally: + agentex_logging._reset_for_tests() + for name, (handlers, propagate) in saved.items(): + existing = logging.Logger.manager.loggerDict.get(name) + if isinstance(existing, logging.Logger): + existing.handlers[:] = handlers + existing.propagate = propagate + + +def _run(handover: bool) -> str: + """One trial in its own process — root-logger state is global and cannot be + isolated within a test session. Returns stdout+stderr.""" + program = textwrap.dedent( + f""" + import logging, sys + from agentex.lib.utils.logging import make_logger, route_agentex_loggers_to_root + + # Exists BEFORE the handover, like any eagerly-imported agentex module. + before = make_logger("agentex.lib.probe.before") + + # Stand in for sgp-obs' pipeline: a single handler on ROOT. + root = logging.getLogger() + root.handlers[:] = [logging.StreamHandler(sys.stdout)] + root.setLevel(logging.INFO) + + if {handover!r}: + route_agentex_loggers_to_root() + + # Created AFTER, like one of the lazily-imported harness modules. + after = make_logger("agentex.lib.probe.after") + + before.info("MARKER-BEFORE") + after.info("MARKER-AFTER") + """ + ) + proc = subprocess.run( + [sys.executable, "-c", program], + capture_output=True, + text=True, + timeout=60, + env={**os.environ, "PYTHONPATH": str(_SRC), "LOG_LEVEL": "INFO", "ENVIRONMENT": "production"}, + ) + assert proc.returncode == 0, proc.stderr[-2000:] + return proc.stdout + proc.stderr + + +class TestEveryRecordIsPrintedOnce: + def test_without_the_handover_everything_doubles(self): + """The bug, pinned. If this ever reads 1, the other two tests below have + stopped proving anything.""" + out = _run(handover=False) + assert out.count("MARKER-BEFORE") == 2 + assert out.count("MARKER-AFTER") == 2 + + def test_a_logger_created_before_the_handover_prints_once(self): + out = _run(handover=True) + assert out.count("MARKER-BEFORE") == 1 + + def test_a_logger_created_after_the_handover_prints_once(self): + """The half a sweep cannot reach: agentex imports harness modules lazily, so + their make_logger runs after init and would attach a fresh duplicate.""" + out = _run(handover=True) + assert out.count("MARKER-AFTER") == 1 + + +class TestTheSweepIsNarrow: + def test_it_clears_an_agentex_logger_that_has_a_handler(self): + lg = logging.getLogger("agentex.lib.probe.sweep") + lg.addHandler(logging.NullHandler()) + assert route_agentex_loggers_to_root() >= 1 + assert lg.handlers == [] + + def test_it_leaves_other_packages_alone(self): + """A third party's handler may be deliberate — which is exactly why sgp-obs + warns about them rather than stripping them.""" + other = logging.getLogger("litellm.probe") + handler = logging.NullHandler() + other.addHandler(handler) + route_agentex_loggers_to_root() + assert other.handlers == [handler] + + def test_it_leaves_a_non_propagating_agentex_logger_alone(self): + """Cut off from root on purpose, so nothing of its reaches the pipeline. + Clearing its handlers would send its records NOWHERE — worse than a duplicate.""" + lg = logging.getLogger("agentex.lib.probe.isolated") + handler = logging.NullHandler() + lg.addHandler(handler) + lg.propagate = False + route_agentex_loggers_to_root() + assert lg.handlers == [handler] + + def test_a_prefix_lookalike_is_not_swept(self): + """`agentexfoo` is a different package, not a child of `agentex`.""" + lg = logging.getLogger("agentexfoo.probe") + handler = logging.NullHandler() + lg.addHandler(handler) + route_agentex_loggers_to_root() + assert lg.handlers == [handler] + + def test_handlers_are_flushed_before_removal(self): + """A buffering handler would otherwise lose whatever it was holding.""" + flushed = [] + + class Recording(logging.NullHandler): + @override + def flush(self): + flushed.append(True) + + lg = logging.getLogger("agentex.lib.probe.flush") + lg.addHandler(Recording()) + route_agentex_loggers_to_root() + assert flushed == [True] + + +class TestMakeLoggerRespectsTheLatch: + def test_it_attaches_nothing_once_the_pipeline_owns_logging(self): + route_agentex_loggers_to_root() + assert make_logger("agentex.lib.probe.after_latch").handlers == [] + + def test_it_still_attaches_when_nothing_owns_logging(self): + """The non-negotiable half: an agent without sgp-obs must log exactly as it + did before any of this existed.""" + agentex_logging._reset_for_tests() + assert make_logger("agentex.lib.probe.no_latch").handlers != [] + + def test_the_level_is_applied_either_way(self, monkeypatch): + """LOG_LEVEL is what agent authors set; letting the pipeline's own threshold + silently replace it would change behaviour nobody asked to change.""" + monkeypatch.setenv("LOG_LEVEL", "DEBUG") + route_agentex_loggers_to_root() + assert make_logger("agentex.lib.probe.level").level == logging.DEBUG