From 8920181ca694e0b2a61a33a00361d62374e74f9d Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 15:30:15 -0700 Subject: [PATCH 01/19] feat(docker): isolate workloads behind the host supervisor Signed-off-by: Drew Newberry --- Cargo.lock | 5 + crates/openshell-driver-docker/Cargo.toml | 11 +- crates/openshell-driver-docker/README.md | 295 +- .../openshell-driver-docker/src/isolation.rs | 154 + crates/openshell-driver-docker/src/lib.rs | 3498 ++++++++++++----- crates/openshell-driver-docker/src/tests.rs | 1722 +++----- docs/reference/gateway-config.mdx | 7 +- docs/reference/sandbox-compute-drivers.mdx | 108 +- e2e/python/test_sandbox_policy.py | 2000 +--------- e2e/python/test_sandbox_venv.py | 54 +- e2e/rust/tests/credential_gating.rs | 172 +- e2e/rust/tests/driver_config_volume.rs | 14 +- e2e/rust/tests/forward_proxy_l7_bypass.rs | 14 +- e2e/rust/tests/gateway_start.rs | 11 +- e2e/rust/tests/local_driver_token_restart.rs | 21 +- e2e/rust/tests/proxy_egress_pipeline.rs | 494 +-- e2e/rust/tests/transparent_tcp.rs | 7 +- rfc/0003-gateway-configuration/README.md | 111 +- 18 files changed, 3832 insertions(+), 4866 deletions(-) create mode 100644 crates/openshell-driver-docker/src/isolation.rs diff --git a/Cargo.lock b/Cargo.lock index fa223f22a9..07a646739e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4098,15 +4098,20 @@ dependencies = [ "clap", "futures", "http 1.4.0", + "libc", "miette", "openshell-core", + "openshell-isolation-interface", "openshell-otel", "openshell-otel-test-support", "opentelemetry", "opentelemetry_sdk", "prost-types", + "rand 0.9.4", + "rustix 1.1.4", "serde", "serde_json", + "sha2 0.10.9", "tar", "temp-env", "tempfile", diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index 0b03fd3880..38051327aa 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } +openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-otel = { path = "../openshell-otel" } opentelemetry = { workspace = true } @@ -38,15 +39,19 @@ miette = { workspace = true } toml = { workspace = true } tower-http = { workspace = true } http = { workspace = true } +rand = { workspace = true } +sha2 = { workspace = true } +rustix = { workspace = true } +libc = "0.2" +tar = "0.4" +tempfile = "3" [dev-dependencies] openshell-otel-test-support = { path = "../openshell-otel-test-support" } opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true, features = ["testing"] } prost-types = { workspace = true } -tar = "0.4" -temp-env = { version = "0.3", features = ["async_closure"] } -tempfile = "3" +temp-env = "0.3" tracing-subscriber = { workspace = true } [lints] diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 10e085ecf0..b8f9cd2011 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -1,148 +1,111 @@ # openshell-driver-docker -Docker-backed compute driver for local OpenShell gateways. +Docker-backed compute driver for local and remote OpenShell gateways. -When the gateway configures `[openshell.gateway.otlp]`, Docker compute-driver -spans export to the same OTLP/gRPC collector with the service name -`openshell-driver-docker`. The in-process driver preserves the gateway trace -context and emits the compute-driver RPC boundary that a standalone driver -would expose. +The driver uses `bollard` to manage sandbox resources through the configured +Docker API socket. When `socket_path` is unset, it selects the first standard +local socket that responds to an API ping. An explicitly selected Docker driver +falls back to `/var/run/docker.sock` when no candidate responds. -`mise run gateway:docker` enables this export only when a local collector is -listening on `127.0.0.1:4317`. Otherwise, it omits the gateway OTLP configuration -so the development gateway does not repeatedly report export failures. - -The standalone `openshell-driver-docker` binary accepts -`OPENSHELL_OTLP_ENDPOINT`. When set, it exports Docker driver spans to that -collector, continues W3C trace context from gateway RPC metadata, and flushes -spans during graceful shutdown. - -The driver manages sandbox containers through the local Docker daemon with the -`bollard` client. It is intended for developer environments where Docker is -already available and running Kubernetes would be unnecessary. - -The driver connects to `[openshell.drivers.docker].socket_path` when configured. -Otherwise, it uses the first standard local Docker socket that responds to an -API ping, which is the same selection mechanism used by gateway auto-detection. -An explicitly selected Docker driver falls back to `/var/run/docker.sock` when -no candidate responds. +When the gateway configures `[openshell.gateway.otlp]`, the in-process driver +exports spans to the same OTLP/gRPC collector as +`openshell-driver-docker`. The standalone driver accepts +`OPENSHELL_OTLP_ENDPOINT`, continues W3C trace context from gateway RPC +metadata, and flushes spans during graceful shutdown. ## Runtime Model -The gateway runs as a host process. The Docker driver creates one container per -sandbox and starts the `openshell-sandbox` supervisor inside that container. The -supervisor then creates the nested sandbox namespace for the agent process. - -## Stop and Start - -Stop stops the managed container without removing it. Docker retains the -container writable layer, attached volumes, labels, token material, and restart -policy. Start starts that same container, so files in the resolved OCI -workspace remain available. A durably stopped sandbox is excluded from -gateway startup recovery and stays stopped across gateway restarts. Delete -continues to force-remove the container and clean up driver-owned material. -Graceful gateway shutdown sends `StopSandbox` for each sandbox whose persisted -phase requires running compute without changing that persisted intent. On -startup, the gateway sends an idempotent `StartSandbox` request for the same -sandboxes, restarting their retained containers. Explicitly stopped sandboxes -remain excluded. - -Before creating the container, the driver inspects the final sandbox image and -captures its immutable image ID, raw OCI `Config.User`, and OCI -`Config.WorkingDir`. Container creation uses that image ID, preventing a -mutable tag from changing between inspection and launch. The supervisor runs as -root, resolves omitted policy identity fields from the image declaration, and -drops only agent children to the resulting identity. Named OCI components -remain names after validation; a missing group is filled with the user's -numeric primary GID. Explicit `process.run_as_user` and -`process.run_as_group` values take precedence independently. - -An absolute OCI working directory becomes the agent workspace. An empty, -root (`/`), or explicit `/sandbox` declaration uses `/sandbox`, which OpenShell -creates when necessary and owns as a compatibility workspace. Any other image -workdir must already exist without symlink components. The completed identity, -including supplementary groups, must already be able to traverse every parent -and write and enter the workdir. OpenShell does not change its ownership or -mode. - -OpenShell deliberately asks the Linux kernel to make this access decision -under the completed sandbox identity instead of reproducing permission rules -from ownership and mode bits. Mode-bit inspection alone can reject authority -granted by a POSIX ACL or overlook a denial imposed by a Linux Security Module -such as SELinux or AppArmor. OpenShell does not configure or otherwise manage -ACLs or LSM policy here; the one-shot validator only observes the kernel's -effective decision. This keeps the no-authority-expansion invariant aligned -with the access the eventual workload will receive without adding a separate, -incomplete permission model to OpenShell. - -Image `VOLUME` declarations must not cover the workdir or one of its parents -because Docker would mount the volume before the supervisor could validate the -immutable image path. -Workdirs under the standard OCI runtime namespaces `/proc`, `/sys`, and `/dev` -are rejected, as are paths that overlap concrete OpenShell control resources. -The workspace is the child cwd and `HOME`. The supervisor starts from `/`, then -reports an invalid workdir as a readiness failure. - -Docker containers join an OpenShell-managed bridge network. The driver injects -`host.openshell.internal` and `host.docker.internal` so supervisors have stable -names for reaching the gateway host. On Docker Desktop, Colima, Rancher -Desktop, OrbStack, and macOS-hosted gateways, those names use Docker's -`host-gateway` alias. The driver requests a separate IPv4 loopback callback -listener when the primary listener does not already cover it. On native Linux -Docker, the gateway also binds the bridge gateway IP so containers can call -back to the host process. +The driver creates two containers for each sandbox: + +- `openshell-sandbox` is PID 1 in the workload container. It owns the workload + process tree, seccomp notification broker, mandatory Landlock baseline, + binary identity, exec/signal/wait/PTY operations, and loopback forwarding. +- `openshell-supervisor` runs in a separate companion container. It owns the + gateway session, policy engine, credentials, interception CA, SSH relay, L7 + inspection, DNS policy, and external upstream connections. + +Both containers are non-root, request no capabilities, and set +no-new-privileges. They share only a driver-created Docker named volume. The +volume carries an authenticated Unix socket and immutable bootstrap material; +it is writable by the sandbox and read-only in the supervisor. + +The workload uses `network_mode=none`. Its seccomp user-notification broker +mediates every supported TCP and DNS operation, attributes it to the calling +binary, and sends the request across the private channel. The supervisor +authorizes the request before it opens an upstream connection. Docker's absent +workload network is the mandatory outer fence if mediation fails or is +bypassed. Only the supervisor companion joins the managed bridge network. + +The driver copies trusted runtime bytes from the configured supervisor image +through the Docker archive API. No workload launch depends on a host bind +mount or a tool supplied by the workload image, so the same path works with +local, remote, and VM-backed Docker daemons. + +## Identity and Workspace + +Before creating the workload, the driver pins the image ID and reads its +passwd/group databases through a stopped metadata container. It resolves the +admitted policy identity, or the image `Config.User` fallback, into one exact +non-root UID, primary GID, and supplementary-group set. Docker launches +`openshell-sandbox` with that identity, and the sandbox uses the same identity +for every canonical and exec process. UID or GID zero and unresolved symbolic +identities are rejected. + +An absolute OCI working directory becomes the workspace. An empty, root (`/`), +or explicit `/sandbox` declaration uses `/sandbox`. Any other workdir must +already exist without symlink components. The resolved identity must be able to +traverse every parent and write and enter the workdir; OpenShell does not +change its ownership or mode. + +Image `VOLUME` declarations and user mounts must not cover the workdir, one of +its parents, or the reserved `/.openshell` runtime/channel tree. OpenShell asks +the kernel to validate access under the final identity, so POSIX ACL and host +LSM decisions remain authoritative. ## Container Contract -The driver-controlled container settings are part of the sandbox security -contract: - | Setting | Purpose | |---|---| -| `user = "0"` | The supervisor needs root inside the container to prepare namespaces, mounts, Landlock, and seccomp. | -| `network_mode = openshell` | Places the supervisor on the managed Docker bridge network. | -| `cap_add` | Grants supervisor-only capabilities required for namespace setup and process inspection. | -| `apparmor=unconfined` | Avoids Docker's default profile blocking required mount operations. | -| `restart_policy = no` | A canonical main-process exit remains terminal and is not silently restarted by Docker. | -| `PidsLimit` | Enforces the sandbox PID budget at the Docker cgroup layer. `[openshell.drivers.docker].sandbox_pids_limit` defaults to `2048`; explicit `0` is invalid. | -| CDI GPU request | Uses opaque `driver_config.cdi_devices` values when set; otherwise selects the requested count of NVIDIA CDI GPUs in round-robin order when daemon CDI support is detected. Docker daemon `/info` can permit `nvidia.com/gpu=all` as a WSL2 all-only compatibility fallback, where it counts as one selectable device. Exact CDI device lists must not contain duplicates and must match the effective GPU count. | -| `policy-dns-transparent-tcp` capability | Declares that the combined Docker supervisor can own namespace-local DNS/TCP capture and coupled workload restart. The shared supervisor still owns DNS eligibility, mappings, authorization, pinned dialing, relaying, and OCSF decisions. The marker is stripped from the workload environment. | - -The agent child process does not retain these supervisor privileges. +| Exact non-root `user` and `group_add` | Gives sandbox and workload the same immutable UID/GID/group identity required for capability-free observation. | +| `cap_drop = ALL`, no `cap_add`, no-new-privileges | Prevents either container from acquiring Linux capabilities. | +| Docker default seccomp and AppArmor profiles | Retains runtime hardening; startup confirmation fails closed if nested seccomp notification is unavailable. | +| `network_mode = none` on the workload | Removes direct external routes. The supervisor companion alone has bridge networking. | +| `restart_policy = no` | Keeps canonical main-process exit terminal. | +| `PidsLimit` | Applies the configured sandbox PID budget. Set `sandbox_pids_limit = 0` to use the runtime default. | +| Private named volume | Carries a per-generation mutual-TLS sandbox/supervisor channel without sharing daemon-host paths. The sandbox consumes its server key at startup; only the supervisor receives the client key. | +| In-memory `/run` tmpfs | Supplies writable runtime state without changing the workload image root filesystem. | +| CDI GPU request | Assigns the exact validated CDI devices requested by driver config or count-based selection. | + +## Stop, Start, and Delete + +Stop terminates the supervisor companion and stops the workload container +without removing it. Docker retains the workload writable layer and attached +volumes. Start stages a fresh sandbox bootstrap bundle, restarts that workload, +and creates a new supervisor companion. A durably stopped sandbox stays stopped +across gateway restarts. + +Delete force-removes both containers, the driver-owned channel volume, and the +host-private topology record. Missing or altered topology and channel resources +fail closed; the driver does not run an older combined-supervisor layout. ## Driver Config Mounts -The gateway forwards the `docker` block from `--driver-config-json` to this -driver. The driver accepts user-supplied `mounts` entries with these Docker -mount types: - -- `bind`: mounts an absolute host path when `[openshell.drivers.docker]` - has `enable_bind_mounts = true`. -- `volume`: mounts an existing Docker named volume. The driver validates that - the volume exists before provisioning and never creates or removes it. - Docker local-driver volumes created with bind options are treated as host - bind mounts and require `enable_bind_mounts = true`. -- `tmpfs`: mounts an in-memory filesystem with optional `options`, - `size_bytes`, and `mode`. - -Host bind mounts are disabled by default because they expose gateway host -paths to sandbox requests. Image mounts are not part of the Docker -driver-config schema. The driver still uses internal bind mounts for -OpenShell-owned supervisor, token, and TLS material. - -Docker `bind` mounts accept `source`, `target`, optional `read_only`, and an -optional `selinux_label` of `shared` (applies `:z`) or `private` (applies -`:Z`) for SELinux-enforcing hosts. Docker `volume` mounts may include -`subpath`. User-supplied bind and volume mounts are read-only by default; set -`read_only: false` to make them writable. Mount `source`, `target`, and -`subpath` values must not contain surrounding whitespace. Mount targets must be -absolute container paths and must not replace or contain the resolved workspace -root. Nested workspace mounts remain valid. Mounts also must not overlap the -configured SSH socket or the reserved `/opt/openshell`, `/etc/openshell`, -`/etc/openshell-tls`, `/run/openshell`, `/run/openshell-sidecar`, and network -namespace roots. - -Example named-volume usage: +The gateway forwards the `docker` block from `--driver-config-json`. Supported +mount types are: + +- `bind`: an absolute daemon-host path, allowed only when + `[openshell.drivers.docker].enable_bind_mounts = true`. +- `volume`: an existing named volume. The driver never creates or removes a + user-supplied volume. Bind-backed local volumes require + `enable_bind_mounts = true`. +- `tmpfs`: an in-memory filesystem with optional size and mode. + +Host bind mounts are disabled by default because they expose daemon-host paths +to sandbox requests. User bind and volume mounts are read-only by default. +Targets must be absolute, normalized paths and cannot overlap the workspace +root or OpenShell control paths. + +Example: ```shell docker volume create openshell-work @@ -152,70 +115,36 @@ openshell sandbox create \ -- claude ``` -## Supervisor Binary Resolution +## Runtime Image -The Docker driver bind-mounts a host-side Linux `openshell-sandbox` binary into -each sandbox container. Resolution order is: - -1. `supervisor_bin` in `[openshell.drivers.docker]`. -2. `supervisor_image` in `[openshell.drivers.docker]`, extracting - `/openshell-sandbox` from that image. -3. A sibling `openshell-sandbox` next to the running `openshell-gateway` binary. -4. A local Linux cargo target build for the Docker daemon architecture. -5. The release-matched default supervisor image, extracting `/openshell-sandbox`. - -Release and Docker-image gateway builds bake the matching supervisor image tag -into the binary at compile time. The default Docker supervisor image is not -`:latest` unless a custom build explicitly sets that tag. +`supervisor_image` must contain `/openshell-sandbox` and +`/openshell-supervisor`. The driver extracts the sandbox binary as bytes and +stages it into the stopped workload. It starts the supervisor binary directly +in the companion container. Release and gateway image builds bake a matching +supervisor image tag into the binary. ## Callback and TLS -`OPENSHELL_ENDPOINT` is injected from the gateway's configured gRPC endpoint. -When no endpoint is configured, the driver uses -`host.openshell.internal:` with the appropriate HTTP or HTTPS -scheme. Set `host_gateway_ip` only when the host has an explicit, locally -assigned address that containers should use for callbacks; package-managed -macOS gateways should leave it unset. - -For HTTPS endpoints, the server certificate must include the endpoint host as a -subject alternative name. Docker sandboxes also need the client TLS bundle -mounted into the container and exposed with: - -- `OPENSHELL_TLS_CA` -- `OPENSHELL_TLS_CERT` -- `OPENSHELL_TLS_KEY` - -HTTP endpoints reject TLS material because the supervisor would not use it. - -## Corporate proxy, SPIFFE, and AppArmor - -`https_proxy`, `no_proxy`, and `proxy_auth_file` in -`[openshell.drivers.docker]` are operator-owned supervisor settings. Docker -passes the proxy URL and bypass list on the supervisor command line and mounts -an optional `user:pass` auth file read-only at a root-only path. Credentials -never appear in container environment or Docker labels. An auth file used with -an `http://` proxy requires `proxy_auth_allow_insecure = true`; an `https://` -proxy protects the Basic-auth header in its TLS session. - -Set `provider_spiffe_workload_api_socket` to an absolute host UNIX socket to -project its dedicated parent directory into the supervisor and set -`OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET` to the guest path. TCP URIs are -rejected for this projection. `app_armor_profile` uses the shared -`RuntimeDefault`, `Unconfined`, or `Localhost/` vocabulary. Docker -uses explicit `Unconfined` by default because the supervisor's namespace mount -setup is incompatible with `docker-default`; requested confined profiles fail -at startup if Docker does not report AppArmor support. +`OPENSHELL_ENDPOINT` and gateway authentication material are injected only into +the supervisor companion. The workload never receives the sandbox JWT, gateway +client TLS key, policy authority, or interception CA private key. -## Environment Ownership +When no endpoint is configured, the driver derives +`host.openshell.internal:`. Native Linux uses the managed bridge +gateway. Docker Desktop and compatible VM-backed daemons use Docker's +`host-gateway` route. A configured HTTPS server certificate must include the +endpoint host in its subject alternative names. -The driver merges template environment and sandbox spec environment first, then -overwrites security-critical keys: +The supervisor owns these security-critical variables: - `OPENSHELL_ENDPOINT` - `OPENSHELL_SANDBOX_ID` - `OPENSHELL_SANDBOX` +- `OPENSHELL_SANDBOX_TOKEN_FILE` - `OPENSHELL_SSH_SOCKET_PATH` - `OPENSHELL_MAIN_PROCESS_SPEC` - TLS path variables when HTTPS is enabled -Do not allow sandbox images or templates to override these values. +Template and sandbox environment is encoded in the protected bootstrap and +exposed only to workload children. Workload input cannot override +security-critical supervisor variables. diff --git a/crates/openshell-driver-docker/src/isolation.rs b/crates/openshell-driver-docker/src/isolation.rs new file mode 100644 index 0000000000..08211cf9c8 --- /dev/null +++ b/crates/openshell-driver-docker/src/isolation.rs @@ -0,0 +1,154 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Docker provisioning for the shared authenticated boundary protocol. +//! +//! Docker owns only the container/socket topology and immutable OCI resource +//! claims. Lifecycle, process, network, identity, and wire behavior live in +//! `openshell-isolation-interface` and `openshell-sandbox`. + +use std::collections::{BTreeMap, HashMap}; +use std::net::IpAddr; +use std::path::PathBuf; + +use openshell_isolation_interface::boundary_protocol::{ + BoundaryClientTls, BoundaryConfig, BoundaryListener, BoundaryServerTls, BoundaryTopology, + BoundaryTransport, +}; +use openshell_isolation_interface::contract::{DriverFenceEvidence, ResolvedWorkloadIdentity}; + +/// Driver-owned inputs that bind one Docker container to one boundary. +pub struct DockerBoundarySpec { + pub boundary_id: String, + pub bootstrap_token: String, + pub generation: String, + pub session_epoch: String, + pub container_id: String, + pub image_identity: String, + pub listener_socket: PathBuf, + pub control_socket: PathBuf, + pub sandbox_tls: BoundaryServerTls, + pub supervisor_tls: BoundaryClientTls, + pub host_gateway_ip: Option, + pub workload_identity: ResolvedWorkloadIdentity, + pub child_env: HashMap, +} + +/// Protected container config and matching host descriptor. +pub struct DockerBoundaryProvisioning { + pub boundary_config: BoundaryConfig, + pub topology: BoundaryTopology, +} + +impl DockerBoundarySpec { + /// Produce both sides of the common protocol from the same immutable + /// Docker coordinates so attach cannot bind a different container. + #[must_use] + pub fn provision(self) -> DockerBoundaryProvisioning { + let resource_claims = BTreeMap::from([ + ("docker.container_id".to_string(), self.container_id), + ("docker.image_identity".to_string(), self.image_identity), + ]); + let driver_fence = DriverFenceEvidence::Docker { + container_id: resource_claims["docker.container_id"].clone(), + network_mode: "none".to_string(), + unexpected_networks: Vec::new(), + }; + DockerBoundaryProvisioning { + boundary_config: BoundaryConfig { + boundary_id: self.boundary_id.clone(), + generation: self.generation.clone(), + session_epoch: self.session_epoch.clone(), + bootstrap_token: self.bootstrap_token.clone(), + listener: BoundaryListener::Unix { + socket_path: self.listener_socket, + tls: self.sandbox_tls, + }, + multiplexed: true, + resource_claims: resource_claims.clone(), + resource_claim_files: BTreeMap::new(), + workload_identity: self.workload_identity.clone(), + driver_fence: driver_fence.clone(), + child_env: self.child_env, + }, + topology: BoundaryTopology { + boundary_id: self.boundary_id, + generation: self.generation, + session_epoch: self.session_epoch, + workload_identity: self.workload_identity, + transport: BoundaryTransport::Unix { + socket_path: self.control_socket, + tls: self.supervisor_tls, + }, + multiplexed: true, + host_gateway_ip: self.host_gateway_ip, + resource_claims, + driver_fence, + bootstrap_token: self.bootstrap_token, + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn provisioning_binds_container_and_image_claims() { + let tls = openshell_isolation_interface::boundary_protocol::generate_boundary_mutual_tls_material() + .unwrap(); + let provisioned = DockerBoundarySpec { + boundary_id: "sandbox-1".to_string(), + bootstrap_token: "a".repeat(64), + generation: "generation-1".to_string(), + session_epoch: "epoch-1".to_string(), + container_id: "sha256:container".to_string(), + image_identity: "sha256:image".to_string(), + listener_socket: PathBuf::from("/run/openshell/boundary/control.sock"), + control_socket: PathBuf::from("/host/control.sock"), + sandbox_tls: BoundaryServerTls { + certificate_chain_path: PathBuf::from("/run/openshell/boundary/server.crt"), + private_key_path: PathBuf::from("/run/openshell/boundary/server.key"), + client_ca_certificate_path: PathBuf::from("/run/openshell/boundary/client-ca.crt"), + }, + supervisor_tls: BoundaryClientTls { + server_name: tls.server_name, + ca_certificate_pem: tls.ca_certificate_pem, + certificate_chain_pem: tls.supervisor_certificate_pem, + private_key_pem: tls.supervisor_private_key_pem, + }, + host_gateway_ip: Some(IpAddr::from([127, 0, 0, 1])), + workload_identity: ResolvedWorkloadIdentity::new( + 1000, + 1000, + Vec::new(), + "image".to_string(), + "sha256:image".to_string(), + ) + .unwrap(), + child_env: HashMap::new(), + } + .provision(); + + assert_eq!( + provisioned.boundary_config.resource_claims, + provisioned.topology.resource_claims + ); + assert_eq!( + provisioned.topology.resource_claims["docker.container_id"], + "sha256:container" + ); + assert_eq!( + provisioned.boundary_config.driver_fence, + provisioned.topology.driver_fence + ); + assert!( + provisioned + .topology + .driver_fence + .validate_for_backend("docker") + .is_ok() + ); + } +} diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index bb3a13687f..397f21f9a3 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -5,23 +5,25 @@ #![allow(clippy::result_large_err)] +mod isolation; pub mod otel_tracing; use bollard::Docker; use bollard::errors::Error as BollardError; use bollard::models::{ ContainerCreateBody, ContainerState, ContainerStateStatusEnum, ContainerSummary, - ContainerSummaryStateEnum, CreateImageInfo, DeviceRequest, EndpointSettings, HostConfig, Mount, - MountTmpfsOptions, MountTypeEnum, MountVolumeOptions, NetworkCreateRequest, NetworkingConfig, - ProgressDetail, SystemInfo, + ContainerSummaryStateEnum, CreateImageInfo, DeviceRequest, HealthConfig, HealthStatusEnum, + HostConfig, Mount, MountTmpfsOptions, MountTypeEnum, MountVolumeOptions, NetworkCreateRequest, + ProgressDetail, SystemInfo, VolumeCreateRequest, }; use bollard::query_parameters::{ CreateContainerOptionsBuilder, CreateImageOptions, DownloadFromContainerOptionsBuilder, - ListContainersOptionsBuilder, RemoveContainerOptionsBuilder, StopContainerOptionsBuilder, + ListContainersOptionsBuilder, ListVolumesOptionsBuilder, LogsOptionsBuilder, + RemoveContainerOptionsBuilder, StopContainerOptionsBuilder, UploadToContainerOptionsBuilder, }; use bytes::Bytes; use futures::{Stream, StreamExt}; -use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS; +use openshell_core::config::{DEFAULT_SANDBOX_PIDS_LIMIT, DEFAULT_STOP_TIMEOUT_SECS}; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ CONDITION_EXITED, CONDITION_RUNTIME_RESTART, CONDITION_WORKSPACE_VALIDATION_FAILED, @@ -57,17 +59,23 @@ use openshell_core::proto::compute::v1::{ use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, }; -use openshell_core::{ - AppArmorProfile, Error, ImagePullPolicy, Result as CoreResult, UpstreamProxyConfig, +use openshell_core::{Error, Result as CoreResult}; +use openshell_isolation_interface::boundary_protocol::{ + BoundaryClientTls, BoundaryServerTls, BoundaryTopology, generate_boundary_mutual_tls_material, }; +use openshell_isolation_interface::contract::ResolvedWorkloadIdentity; use opentelemetry::trace::TraceContextExt as _; +use sha2::{Digest as _, Sha256}; use std::collections::{HashMap, HashSet}; +use std::fmt::Write as _; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +#[cfg(unix)] use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; -use tokio::sync::{Mutex, broadcast, mpsc}; +use tokio::sync::{Mutex, broadcast, mpsc, oneshot}; use tokio::task::JoinHandle; use tokio_stream::wrappers::ReceiverStream; use tonic::{Request, Response, Status}; @@ -78,17 +86,43 @@ use url::Url; const WATCH_BUFFER: usize = 128; const WATCH_POLL_INTERVAL: Duration = Duration::from_secs(2); const WATCH_POLL_MAX_BACKOFF: Duration = Duration::from_secs(30); - -const SUPERVISOR_MOUNT_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; -const TLS_CA_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CA_MOUNT_PATH; -const TLS_CERT_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CERT_MOUNT_PATH; -const TLS_KEY_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_KEY_MOUNT_PATH; -const SANDBOX_TOKEN_MOUNT_PATH: &str = openshell_core::driver_utils::SANDBOX_TOKEN_MOUNT_PATH; -const UPSTREAM_PROXY_AUTH_MOUNT_PATH: &str = - openshell_core::driver_utils::UPSTREAM_PROXY_AUTH_MOUNT_PATH; -const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str = - openshell_core::driver_utils::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR; -const SUPERVISOR_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; +const SUPERVISOR_READY_TIMEOUT: Duration = Duration::from_secs(90); +// The gateway closes a supervisor session as soon as it commits a sandbox to +// Stopping, just before the compute-driver StopSandbox RPC arrives. Give that +// request a bounded opportunity to mark the control exit intentional before +// publishing a fail-closed runtime error. +const SUPERVISOR_INTENTIONAL_SHUTDOWN_GRACE: Duration = Duration::from_secs(1); +const SUPERVISOR_HEALTH_INTERVAL_NS: i64 = 250_000_000; +const SUPERVISOR_HEALTH_TIMEOUT_NS: i64 = 2_000_000_000; +const SUPERVISOR_HEALTH_START_PERIOD_NS: i64 = 60_000_000_000; + +const SANDBOX_BINARY_PATH: &str = "/.openshell/runtime/openshell-sandbox"; +const SUPERVISOR_IMAGE_CONTROL_BINARY_PATH: &str = "/openshell-supervisor"; +const SUPERVISOR_HEALTH_SOCKET_PATH: &str = "/run/openshell/health.sock"; +const SUPERVISOR_UID: u32 = 65_534; +const SUPERVISOR_GID: u32 = 65_534; +const BOUNDARY_MOUNT_PATH: &str = "/.openshell/channel"; +const BOUNDARY_CONFIG_MOUNT_PATH: &str = "/.openshell/channel/sandbox/bootstrap.json"; +const BOUNDARY_SOCKET_MOUNT_PATH: &str = "/.openshell/channel/sandbox/control.sock"; +const BOUNDARY_CERTIFICATE_MOUNT_PATH: &str = "/.openshell/channel/sandbox/server.crt"; +const BOUNDARY_PRIVATE_KEY_MOUNT_PATH: &str = "/.openshell/channel/sandbox/server.key"; +const BOUNDARY_CLIENT_CA_MOUNT_PATH: &str = "/.openshell/channel/sandbox/client-ca.crt"; +const SUPERVISOR_STATE_MOUNT_PATH: &str = "/.openshell/channel/supervisor"; +const DRIVER_ADMITTED_BACKEND: &str = "docker"; +const LABEL_ISOLATION_TOPOLOGY: &str = "openshell.ai/isolation-topology"; +const LABEL_ISOLATION_TOPOLOGY_CAPABILITY_FREE: &str = "capability-free"; +const LABEL_ISOLATION_ROLE: &str = "openshell.ai/isolation-role"; +const LABEL_ISOLATION_ROLE_SANDBOX: &str = "sandbox"; +const LABEL_ISOLATION_ROLE_SUPERVISOR: &str = "supervisor"; +const LABEL_ISOLATION_ROLE_STAGING: &str = "staging"; +const LABEL_ISOLATION_ROLE_IDENTITY: &str = "identity"; +const TOPOLOGY_PAYLOAD_FILE: &str = "topology.payload"; +const MAIN_PROCESS_SPEC_FILE: &str = "main-process.json"; +const WORKSPACE_ROOT_FILE: &str = "workspace-root"; +const BOUNDARY_CONFIG_FILE: &str = "boundary-bootstrap.json"; +const BOUNDARY_CERTIFICATE_FILE: &str = "boundary-server.crt"; +const BOUNDARY_PRIVATE_KEY_FILE: &str = "boundary-server.key"; +const BOUNDARY_CLIENT_CA_FILE: &str = "boundary-client-ca.crt"; const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; const DOCKER_NETWORK_DRIVER: &str = "bridge"; @@ -128,20 +162,16 @@ pub struct DockerComputeConfig { pub default_image: String, /// Image pull policy for sandbox images. - pub image_pull_policy: ImagePullPolicy, + pub image_pull_policy: String, - /// Value of the `openshell.sandbox_namespace` label applied to Docker sandboxes. - pub sandbox_label: String, + /// Namespace label applied to Docker sandboxes. + pub sandbox_namespace: String, /// Gateway gRPC endpoint the sandbox connects back to. pub grpc_endpoint: String, - /// Optional override for the Linux `openshell-sandbox` binary mounted into containers. - pub supervisor_bin: Option, - - /// Optional image used to extract the Linux `openshell-sandbox` binary. - /// Ignored when `supervisor_bin` is set. See `resolve_supervisor_bin` for - /// the full resolution order. + /// Image containing the trusted `openshell-sandbox` and + /// `openshell-supervisor` binaries. pub supervisor_image: Option, /// Host-side CA certificate for Docker sandbox mTLS. @@ -159,66 +189,15 @@ pub struct DockerComputeConfig { /// Host gateway IP used for sandbox host aliases. pub host_gateway_ip: String, - /// Unix socket path the in-container supervisor bridges relay traffic to. - pub ssh_socket_path: String, - /// Container cgroup PID limit for Docker-managed sandboxes. /// - /// Omit the field to use `OpenShell`'s 2048-process sandbox limit. Explicit - /// zero is invalid. - #[serde( - default = "openshell_core::config::default_sandbox_pids_limit", - skip_serializing_if = "Option::is_none" - )] - pub sandbox_pids_limit: Option, + /// Set to `0` to leave Docker's runtime/default PID limit unchanged. + pub sandbox_pids_limit: i64, /// Allow sandbox requests to attach host bind mounts through /// `template.driver_config`. #[serde(default)] pub enable_bind_mounts: bool, - - /// Corporate forward-proxy settings supplied to the supervisor on argv. - /// The flattened fields retain the common `https_proxy`, `no_proxy`, and - /// `proxy_auth_*` gateway TOML contract. - #[serde(flatten)] - pub upstream_proxy: UpstreamProxyConfig, - - /// Host UNIX socket to project into sandbox supervisors for provider - /// SPIFFE token exchange. - pub provider_spiffe_workload_api_socket: Option, - - /// `AppArmor` confinement requested for sandbox containers. The explicit - /// default preserves the prior supervisor-compatible Docker behavior. - #[serde(skip_serializing_if = "Option::is_none")] - pub app_armor_profile: Option, -} - -impl DockerComputeConfig { - /// Validate startup configuration without connecting to Docker. - pub fn validate_configuration(&self, gateway_bind_address: SocketAddr) -> CoreResult<()> { - if let Some(socket_path) = self.socket_path.as_deref() - && socket_path.to_str().is_none() - { - return Err(Error::config(format!( - "Docker socket path is not valid UTF-8: {}", - socket_path.display() - ))); - } - validate_sandbox_pids_limit(self.sandbox_pids_limit)?; - validate_image_pull_policy(self.image_pull_policy)?; - self.upstream_proxy.validate().map_err(Error::config)?; - if let Some(socket) = self.provider_spiffe_workload_api_socket.as_deref() { - openshell_core::driver_utils::validate_provider_spiffe_unix_socket(socket) - .map_err(Error::config)?; - } - parse_optional_host_gateway_ip(&self.host_gateway_ip)?; - if gateway_bind_address.port() == 0 { - return Err(Error::config( - "docker compute driver requires a fixed non-zero gateway bind port", - )); - } - Ok(()) - } } impl Default for DockerComputeConfig { @@ -226,22 +205,17 @@ impl Default for DockerComputeConfig { Self { socket_path: None, default_image: openshell_core::image::default_sandbox_image(), - image_pull_policy: ImagePullPolicy::default(), - sandbox_label: "default".to_string(), + image_pull_policy: String::new(), + sandbox_namespace: "default".to_string(), grpc_endpoint: String::new(), - supervisor_bin: None, supervisor_image: None, guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), host_gateway_ip: String::new(), - ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), - sandbox_pids_limit: openshell_core::config::default_sandbox_pids_limit(), + sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, enable_bind_mounts: false, - upstream_proxy: UpstreamProxyConfig::default(), - provider_spiffe_workload_api_socket: None, - app_armor_profile: Some(AppArmorProfile::Unconfined), } } } @@ -256,24 +230,22 @@ pub(crate) struct DockerGuestTlsPaths { #[derive(Debug, Clone)] struct DockerDriverRuntimeConfig { default_image: String, - image_pull_policy: ImagePullPolicy, - sandbox_label: String, - grpc_endpoint: String, - network_name: String, + image_pull_policy: String, + sandbox_namespace: String, gateway_route: DockerGatewayRoute, gateway_callback_bind_address: Option, - ssh_socket_path: String, stop_timeout_secs: u32, log_level: String, - supervisor_bin: PathBuf, + sandbox_binary: Arc>, + supervisor_image_id: String, + network_name: String, + supervisor_grpc_endpoint: String, + gateway_tls_server_name: Option, guest_tls: Option, daemon_version: String, gpu: DockerGpuRuntimeCapabilities, - sandbox_pids_limit: Option, + sandbox_pids_limit: i64, enable_bind_mounts: bool, - upstream_proxy: UpstreamProxyConfig, - provider_spiffe_workload_api_socket: Option, - app_armor_profile: Option, } #[derive(Debug, Clone, Copy)] @@ -284,10 +256,7 @@ struct DockerGpuRuntimeCapabilities { #[derive(Debug, Clone, PartialEq, Eq)] enum DockerGatewayRoute { - Bridge { - bind_address: SocketAddr, - host_alias_ip: IpAddr, - }, + Bridge { bind_address: SocketAddr }, HostGateway, } @@ -299,6 +268,31 @@ pub struct DockerComputeDriver { pending: Arc>>, gpu_selector: Arc, lifecycle_event_fences: DockerLifecycleEventFences, + control_processes: Arc>>, + runtime_failures: Arc>>, +} + +struct DockerControlProcess { + shutdown: Option>, + intentional_shutdown: Arc, + task: JoinHandle<()>, +} + +#[derive(Clone)] +struct DockerRuntimeFailure { + reason: &'static str, + message: String, +} + +#[derive(Clone)] +struct DockerRuntimeFailureContext { + docker: Arc, + events: broadcast::Sender, + failures: Arc>>, + sandbox: DriverSandbox, + sandbox_namespace: String, + container_id: String, + stop_timeout_secs: u32, } /// Per-sandbox container exit timestamps that fence snapshots from an earlier run. @@ -316,6 +310,7 @@ struct DockerLifecycleEventFences { struct DockerLifecycleFenceState { previous_finished_at: HashMap, starts_in_progress: HashSet, + stops_requested: HashSet, } impl DockerLifecycleEventFences { @@ -343,6 +338,46 @@ impl DockerLifecycleEventFences { .contains(sandbox_id) } + fn request_stop(&self, sandbox_id: &str, sandbox_name: &str) { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !sandbox_id.is_empty() { + state.stops_requested.insert(format!("id:{sandbox_id}")); + } + if !sandbox_name.is_empty() { + state.stops_requested.insert(format!("name:{sandbox_name}")); + } + } + + fn clear_stop(&self, sandbox_id: &str, sandbox_name: &str) { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !sandbox_id.is_empty() { + state.stops_requested.remove(&format!("id:{sandbox_id}")); + } + if !sandbox_name.is_empty() { + state + .stops_requested + .remove(&format!("name:{sandbox_name}")); + } + } + + fn stop_requested(&self, sandbox_id: &str, sandbox_name: &str) -> bool { + let state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + (!sandbox_id.is_empty() && state.stops_requested.contains(&format!("id:{sandbox_id}"))) + || (!sandbox_name.is_empty() + && state + .stops_requested + .contains(&format!("name:{sandbox_name}"))) + } + fn record_previous_exit(&self, sandbox_id: &str, finished_at: Option<&str>) { let mut state = self .state @@ -369,13 +404,17 @@ impl DockerLifecycleEventFences { .cloned() } - fn remove(&self, sandbox_id: &str) { + fn remove(&self, sandbox_id: &str, sandbox_name: &str) { let mut state = self .state .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); state.previous_finished_at.remove(sandbox_id); state.starts_in_progress.remove(sandbox_id); + state.stops_requested.remove(&format!("id:{sandbox_id}")); + state + .stops_requested + .remove(&format!("name:{sandbox_name}")); } } @@ -398,6 +437,195 @@ struct DockerImageMetadata { volumes: Vec, } +#[derive(Debug, Clone, PartialEq, Eq)] +struct DockerPasswdEntry { + name: String, + uid: u32, + gid: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct DockerGroupEntry { + name: String, + gid: u32, + members: Vec, +} + +fn parse_docker_passwd(bytes: &[u8]) -> Result, Status> { + let contents = std::str::from_utf8(bytes).map_err(|error| { + Status::failed_precondition(format!("image /etc/passwd is not UTF-8: {error}")) + })?; + let mut entries = Vec::new(); + for (index, line) in contents.lines().enumerate() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let fields = line.split(':').collect::>(); + if fields.len() < 4 || fields[0].is_empty() { + return Err(Status::failed_precondition(format!( + "image /etc/passwd line {} is malformed", + index + 1 + ))); + } + let uid = fields[2].parse::().map_err(|_| { + Status::failed_precondition(format!( + "image /etc/passwd line {} has an invalid UID", + index + 1 + )) + })?; + let gid = fields[3].parse::().map_err(|_| { + Status::failed_precondition(format!( + "image /etc/passwd line {} has an invalid GID", + index + 1 + )) + })?; + entries.push(DockerPasswdEntry { + name: fields[0].to_string(), + uid, + gid, + }); + } + Ok(entries) +} + +fn parse_docker_group(bytes: &[u8]) -> Result, Status> { + let contents = std::str::from_utf8(bytes).map_err(|error| { + Status::failed_precondition(format!("image /etc/group is not UTF-8: {error}")) + })?; + let mut entries = Vec::new(); + for (index, line) in contents.lines().enumerate() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let fields = line.split(':').collect::>(); + if fields.len() < 4 || fields[0].is_empty() { + return Err(Status::failed_precondition(format!( + "image /etc/group line {} is malformed", + index + 1 + ))); + } + let gid = fields[2].parse::().map_err(|_| { + Status::failed_precondition(format!( + "image /etc/group line {} has an invalid GID", + index + 1 + )) + })?; + entries.push(DockerGroupEntry { + name: fields[0].to_string(), + gid, + members: fields[3] + .split(',') + .filter(|member| !member.is_empty()) + .map(str::to_string) + .collect(), + }); + } + Ok(entries) +} + +fn resolve_numeric_or_named_user<'a>( + selector: &str, + passwd: &'a [DockerPasswdEntry], +) -> Result<(u32, Option<&'a DockerPasswdEntry>), Status> { + if let Ok(uid) = selector.parse::() { + return Ok((uid, passwd.iter().find(|entry| entry.uid == uid))); + } + let entry = passwd + .iter() + .find(|entry| entry.name == selector) + .ok_or_else(|| { + Status::failed_precondition(format!( + "workload user '{selector}' does not exist in the pinned image" + )) + })?; + Ok((entry.uid, Some(entry))) +} + +fn resolve_numeric_or_named_group( + selector: &str, + groups: &[DockerGroupEntry], +) -> Result { + if let Ok(gid) = selector.parse::() { + return Ok(gid); + } + groups + .iter() + .find(|entry| entry.name == selector) + .map(|entry| entry.gid) + .ok_or_else(|| { + Status::failed_precondition(format!( + "workload group '{selector}' does not exist in the pinned image" + )) + }) +} + +fn resolve_docker_identity_from_accounts( + sandbox: &DriverSandbox, + image: &DockerImageMetadata, + passwd_bytes: &[u8], + group_bytes: &[u8], +) -> Result { + let passwd = parse_docker_passwd(passwd_bytes)?; + let groups = parse_docker_group(group_bytes)?; + let request = sandbox + .spec + .as_ref() + .and_then(|spec| spec.workload_identity.as_ref()); + let requested_user = request.map_or("", |request| request.user.trim()); + let requested_group = request.map_or("", |request| request.group.trim()); + let (image_user, image_group) = image.user.split_once(':').unwrap_or((&image.user, "")); + let user_selector = if requested_user.is_empty() { + image_user.trim() + } else { + requested_user + }; + if user_selector.is_empty() { + return Err(Status::failed_precondition( + "the pinned image defaults to root; configure a non-root process.run_as_user", + )); + } + let (uid, passwd_entry) = resolve_numeric_or_named_user(user_selector, &passwd)?; + let username = passwd_entry.map(|entry| entry.name.as_str()); + let group_selector = if requested_group.is_empty() { + image_group.trim() + } else { + requested_group + }; + let gid = if group_selector.is_empty() { + passwd_entry.map(|entry| entry.gid).ok_or_else(|| { + Status::failed_precondition(format!( + "numeric workload UID {uid} has no passwd entry; configure process.run_as_group" + )) + })? + } else { + resolve_numeric_or_named_group(group_selector, &groups)? + }; + let supplementary_gids = username.map_or_else(Vec::new, |username| { + groups + .iter() + .filter(|entry| { + entry.gid != gid && entry.members.iter().any(|member| member == username) + }) + .map(|entry| entry.gid) + .collect() + }); + let source = if !requested_user.is_empty() || !requested_group.is_empty() { + "policy" + } else { + "image" + }; + ResolvedWorkloadIdentity::new( + uid, + gid, + supplementary_gids, + source.to_string(), + image.id.clone(), + ) + .map_err(|error| Status::failed_precondition(error.to_string())) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] struct DockerResourceLimits { nano_cpus: Option, @@ -543,7 +771,6 @@ impl DockerComputeDriver { gateway_log_level: &str, docker_config: &DockerComputeConfig, ) -> CoreResult { - docker_config.validate_configuration(gateway_bind_address)?; let socket_path = docker_config .socket_path .clone() @@ -576,9 +803,13 @@ impl DockerComputeDriver { cdi_supported, wsl_all_gpu_fallback_enabled, }; - validate_docker_proxy_auth_file(&docker_config.upstream_proxy)?; - validate_docker_app_armor_profile(docker_config.app_armor_profile.as_ref(), &info)?; + validate_sandbox_pids_limit(docker_config.sandbox_pids_limit)?; let gateway_port = gateway_bind_address.port(); + if gateway_port == 0 { + return Err(Error::config( + "docker compute driver requires a fixed non-zero gateway bind port", + )); + } let network_name = docker_network_name(docker_config); let bridge_gateway_ip = ensure_bridge_network(&docker, &network_name).await?; let host_gateway_ip = parse_optional_host_gateway_ip(&docker_config.host_gateway_ip)?; @@ -594,39 +825,67 @@ impl DockerComputeDriver { docker_guest_tls_configured(&docker_config), ); } - let grpc_endpoint = docker_container_openshell_endpoint( - &docker_config.grpc_endpoint, - HOST_OPENSHELL_INTERNAL, - gateway_port, + let host_grpc_endpoint = + docker_host_openshell_endpoint(&docker_config.grpc_endpoint, &gateway_route)?; + let original_gateway_url = Url::parse(&docker_config.grpc_endpoint).map_err(|error| { + Error::config(format!( + "invalid docker grpc_endpoint '{}': {error}", + docker_config.grpc_endpoint + )) + })?; + let host_gateway_url = Url::parse(&host_grpc_endpoint).map_err(|error| { + Error::config(format!( + "invalid normalized Docker host grpc_endpoint '{host_grpc_endpoint}': {error}" + )) + })?; + let gateway_tls_server_name = (original_gateway_url.scheme() == "https" + && original_gateway_url.host_str() != host_gateway_url.host_str()) + .then(|| { + original_gateway_url + .host_str() + .unwrap_or_default() + .to_string() + }); + let supervisor_grpc_endpoint = match &gateway_route { + DockerGatewayRoute::Bridge { .. } => host_grpc_endpoint, + DockerGatewayRoute::HostGateway => docker_config.grpc_endpoint.clone(), + }; + let supervisor_image = docker_config + .supervisor_image + .clone() + .unwrap_or_else(openshell_core::config::default_supervisor_image); + let supervisor_image_id = + ensure_supervisor_container_image(&docker, &supervisor_image).await?; + let sandbox_binary = Arc::new( + extract_supervisor_binary_bytes(&docker, &supervisor_image_id) + .await + .map_err(|error| { + Error::config(format!( + "failed to load trusted sandbox binary from Docker image '{supervisor_image}': {error}" + )) + })?, ); - let daemon_arch = normalize_docker_arch(version.arch.as_deref().unwrap_or_default()); - let supervisor_bin = resolve_supervisor_bin(&docker, &docker_config, &daemon_arch).await?; let guest_tls = docker_guest_tls_paths(&docker_config)?; - let driver = Self { docker: Arc::new(docker), config: DockerDriverRuntimeConfig { default_image: docker_config.default_image.clone(), - image_pull_policy: docker_config.image_pull_policy, - sandbox_label: docker_config.sandbox_label.clone(), - grpc_endpoint, - network_name, + image_pull_policy: docker_config.image_pull_policy.clone(), + sandbox_namespace: docker_config.sandbox_namespace.clone(), gateway_route, gateway_callback_bind_address, - ssh_socket_path: docker_config.ssh_socket_path.clone(), stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, log_level: gateway_log_level.to_string(), - supervisor_bin, + sandbox_binary, + supervisor_image_id, + network_name, + supervisor_grpc_endpoint, + gateway_tls_server_name, guest_tls, daemon_version: version.version.unwrap_or_else(|| "unknown".to_string()), gpu, sandbox_pids_limit: docker_config.sandbox_pids_limit, enable_bind_mounts: docker_config.enable_bind_mounts, - upstream_proxy: docker_config.upstream_proxy.clone(), - provider_spiffe_workload_api_socket: docker_config - .provider_spiffe_workload_api_socket - .clone(), - app_armor_profile: docker_config.app_armor_profile.clone(), }, events: broadcast::channel(WATCH_BUFFER).0, pending: Arc::new(Mutex::new(HashMap::new())), @@ -635,8 +894,19 @@ impl DockerComputeDriver { gpu.wsl_all_gpu_fallback_enabled, )), lifecycle_event_fences: DockerLifecycleEventFences::default(), + control_processes: Arc::new(Mutex::new(HashMap::new())), + runtime_failures: Arc::new(Mutex::new(HashMap::new())), }; + Box::pin(driver.reconcile_runtime_resources_at_startup()) + .await + .map_err(|error| { + Error::config(format!( + "failed to reconcile Docker isolation resources: {}", + error.message() + )) + })?; + let poll_driver = driver.clone(); tokio::spawn(async move { poll_driver.poll_loop().await; @@ -842,21 +1112,93 @@ impl DockerComputeDriver { .map_err(docker_gpu_selection_status) } + async fn resolve_docker_workload_identity( + &self, + sandbox: &DriverSandbox, + image: &DockerImageMetadata, + ) -> Result { + let container_name = format!("{}-identity", temp_extract_container_name()); + self.docker + .create_container( + Some( + CreateContainerOptionsBuilder::default() + .name(container_name.as_str()) + .build(), + ), + ContainerCreateBody { + image: Some(image.id.clone()), + labels: Some(docker_auxiliary_container_labels( + sandbox, + &self.config, + LABEL_ISOLATION_ROLE_IDENTITY, + )), + ..Default::default() + }, + ) + .await + .map_err(|error| { + Status::failed_precondition(format!( + "create Docker identity resolver container: {error}" + )) + })?; + + let result = async { + let passwd = + download_path_from_container(&self.docker, &container_name, "/etc/passwd", true) + .await + .map_err(|error| { + Status::failed_precondition(format!( + "read immutable image /etc/passwd for workload identity: {error}" + )) + })?; + let group = + download_path_from_container(&self.docker, &container_name, "/etc/group", true) + .await + .map_err(|error| { + Status::failed_precondition(format!( + "read immutable image /etc/group for workload identity: {error}" + )) + })?; + resolve_docker_identity_from_accounts(sandbox, image, &passwd, &group) + } + .await; + + if let Err(error) = self + .docker + .remove_container( + &container_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await + { + warn!( + container = container_name, + %error, + "Failed to remove Docker identity resolver container" + ); + } + result + } + async fn get_sandbox_snapshot( &self, sandbox_id: &str, sandbox_name: &str, ) -> Result, Status> { + if let Some(pending) = self.pending_snapshot(sandbox_id, sandbox_name).await { + return Ok(Some(pending)); + } let container = self .find_managed_container_summary(sandbox_id, sandbox_name) .await?; - if let Some(sandbox) = + if let Some(mut sandbox) = container.and_then(|summary| sandbox_from_container_summary(&summary)) { + self.apply_runtime_failure(&mut sandbox).await; return Ok(Some(sandbox)); } - self.pending_snapshot(sandbox_id, sandbox_name).await + Ok(None) } async fn current_snapshots(&self) -> Result, Status> { @@ -866,36 +1208,49 @@ impl DockerComputeDriver { let Some(mut sandbox) = sandbox_from_container_summary(summary) else { continue; }; - // Docker's list summary carries no exit code, so an exited - // container is reported as the generic terminal `ContainerExited`. - // Inspect it to tell a machine/daemon-restart signal kill apart - // from an ordinary application exit, mirroring the Podman driver, - // so startup recovery can revive restart victims while leaving - // crashes terminal. - if summary.state == Some(ContainerSummaryStateEnum::EXITED) - && let Some(container_id) = summary.id.as_deref() - { + if let Some(container_id) = summary.id.as_deref() { match self.docker.inspect_container(container_id, None).await { - Ok(inspected) => { + Ok(inspected) if summary.state == Some(ContainerSummaryStateEnum::EXITED) => { + // Docker's list summary carries no exit code. Inspect + // exited containers so daemon-restart kills remain + // distinguishable from terminal application exits. if let Some(state) = inspected.state.as_ref() { apply_docker_exit_classification(&mut sandbox, state); } } + Ok(inspected) if summary.state == Some(ContainerSummaryStateEnum::RUNNING) => { + if let Err(status) = validate_docker_outer_fence(&inspected) { + let context = self + .control_failure_context(sandbox.clone(), container_id.to_string()); + handle_docker_runtime_failure( + context, + "OuterFenceViolation", + status.message().to_string(), + ) + .await; + } + } + Ok(_) => {} Err(err) => { debug!( container_id, error = %err, - "Could not inspect exited Docker container to classify its exit" + "Could not inspect Docker sandbox container during reconciliation" ); } } } + self.apply_runtime_failure(&mut sandbox).await; container_sandboxes.push(sandbox); } - let mut by_id = self.pending_snapshot_map().await; - for sandbox in container_sandboxes { - by_id.insert(sandbox.id.clone(), sandbox); - } + let mut by_id = container_sandboxes + .into_iter() + .map(|sandbox| (sandbox.id.clone(), sandbox)) + .collect::>(); + // Provisioning state is authoritative until the supervisor has + // attached to both the sandbox and gateway. A running workload + // container alone is not a usable sandbox. + by_id.extend(self.pending_snapshot_map().await); let mut sandboxes = by_id.into_values().collect::>(); sandboxes.sort_by(|left, right| left.id.cmp(&right.id)); Ok(sandboxes) @@ -932,7 +1287,7 @@ impl DockerComputeDriver { ); self.publish_sandbox_snapshot(pending_sandbox_snapshot( sandbox, - &self.config.sandbox_label, + &self.config.sandbox_namespace, provisioning_condition(), false, )); @@ -944,7 +1299,7 @@ impl DockerComputeDriver { let provisioning_span = provisioning_span(&parent, sandbox, &image); let task = tokio::spawn( async move { - driver.provision_sandbox(sandbox_for_task).await; + Box::pin(driver.provision_sandbox(sandbox_for_task)).await; } .instrument(provisioning_span), ); @@ -960,9 +1315,19 @@ impl DockerComputeDriver { } async fn provision_sandbox(&self, sandbox: DriverSandbox) { - match self.provision_sandbox_inner(&sandbox).await { + match Box::pin(self.provision_sandbox_inner(&sandbox)).await { Ok(()) => { self.clear_pending_sandbox(&sandbox.id).await; + if let Err(error) = self + .publish_container_snapshot(&sandbox.id, &sandbox.name) + .await + { + warn!( + sandbox_id = %sandbox.id, + %error, + "Failed to publish Docker sandbox snapshot after provisioning" + ); + } } Err(failure) => { self.fail_pending_sandbox(&sandbox, &failure).await; @@ -996,40 +1361,83 @@ impl DockerComputeDriver { image.ref = %template.image, )) .await?; - let token_file_created = write_sandbox_token_file(sandbox, &self.config) + let workload_identity = self + .resolve_docker_workload_identity(sandbox, &image) + .await + .map_err(|status| { + DockerProvisioningFailure::new("IdentityResolutionFailed", status.message()) + })?; + prepare_docker_boundary_state_dir(sandbox, &self.config).map_err(|status| { + DockerProvisioningFailure::new("BoundaryStateCreateFailed", status.message()) + })?; + create_docker_channel_volume(&self.docker, sandbox, &self.config) .await .map_err(|status| { - DockerProvisioningFailure::new("SandboxTokenWriteFailed", status.message()) + cleanup_docker_boundary_state(sandbox, &self.config); + DockerProvisioningFailure::new("BoundaryChannelCreateFailed", status.message()) })?; + let token_file_created = match write_sandbox_token_file(sandbox, &self.config).await { + Ok(created) => created, + Err(status) => { + let _ = remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "SandboxTokenWriteFailed", + status.message(), + )); + } + }; + if !token_file_created { + let _ = + remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config).await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "SandboxTokenWriteFailed", + "Docker control mode requires a gateway sandbox token", + )); + } let container_name = container_name_for_sandbox(sandbox); - let gpu_devices = self + let gpu_devices = match self .resolve_gpu_cdi_devices( validated.gpu_requirements, &validated.driver_config, CdiGpuDefaultSelector::next_device_ids, ) .await - .map_err(|status| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) - })?; - let create_body = build_container_create_body_for_image( + { + Ok(devices) => devices, + Err(status) => { + let _ = remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "ContainerCreateFailed", + status.message(), + )); + } + }; + let create_body = match build_container_create_body_for_image( sandbox, &self.config, &validated.driver_config, gpu_devices.as_deref(), &image, - ) - .map_err(|status| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); + &workload_identity, + ) { + Ok(body) => body, + Err(status) => { + let _ = remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "ContainerCreateFailed", + status.message(), + )); } - DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) - })?; - async { + }; + let create_result = async { openshell_otel::record_error_result( self.docker .create_container( @@ -1040,16 +1448,7 @@ impl DockerComputeDriver { ), create_body, ) - .await - .map_err(|err| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - DockerProvisioningFailure::from_status( - "ContainerCreateFailed", - create_status_from_docker_error("create docker sandbox container", err), - ) - }), + .await, ) } .instrument(tracing::info_span!( @@ -1059,7 +1458,56 @@ impl DockerComputeDriver { sandbox.id = %sandbox.id, container.name = %container_name, )) - .await?; + .await; + let created = match create_result { + Ok(created) => created, + Err(error) => { + let _ = remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::from_status( + "ContainerCreateFailed", + create_status_from_docker_error("create docker sandbox container", error), + )); + } + }; + let inspected = match self.docker.inspect_container(&created.id, None).await { + Ok(inspected) => inspected, + Err(error) => { + let _ = self + .docker + .remove_container( + &created.id, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + let _ = remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::from_status( + "OuterFenceInspectFailed", + internal_status("inspect Docker sandbox outer fence", error), + )); + } + }; + let outer_fence_error = validate_docker_outer_fence(&inspected).err(); + drop(inspected); + if let Some(status) = outer_fence_error { + let _ = self + .docker + .remove_container( + &created.id, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + let _ = + remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config).await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::from_status( + "OuterFenceRejected", + status, + )); + } self.publish_docker_progress( &sandbox.id, "Created", @@ -1067,20 +1515,49 @@ impl DockerComputeDriver { HashMap::from([("container_name".to_string(), container_name.clone())]), ); - let start_result = async { - openshell_otel::record_error_result( - self.docker.start_container(&container_name, None).await, - ) - } - .instrument(tracing::info_span!( - "docker.start_container", - otel.name = "docker.start_container", - otel.status_code = tracing::field::Empty, - sandbox.id = %sandbox.id, - container.name = %container_name, - )) - .await; - if let Err(err) = start_result { + let topology = match prepare_docker_boundary_files( + &self.docker, + sandbox, + &self.config, + &created.id, + &image, + &workload_identity, + ) + .await + { + Ok(topology) => topology, + Err(status) => { + let _ = self + .docker + .remove_container( + &container_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + let _ = remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "BoundaryConfigWriteFailed", + status.message(), + )); + } + }; + + let start_result = async { + openshell_otel::record_error_result( + self.docker.start_container(&container_name, None).await, + ) + } + .instrument(tracing::info_span!( + "docker.start_container", + otel.name = "docker.start_container", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox.id, + container.name = %container_name, + )) + .await; + if let Err(err) = start_result { let cleanup = self .docker .remove_container( @@ -1096,32 +1573,365 @@ impl DockerComputeDriver { "Failed to clean up Docker container after start failure" ); } - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } + cleanup_docker_boundary_state(sandbox, &self.config); + let _ = + remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config).await; return Err(DockerProvisioningFailure::from_status( "ContainerStartFailed", create_status_from_docker_error("start docker sandbox container", err), )); } + self.clear_runtime_failure(&sandbox.id).await; + let failure_context = self.control_failure_context(sandbox.clone(), created.id.clone()); + let control = match spawn_docker_control_process( + &self.docker, + sandbox, + &self.config, + &topology, + failure_context, + ) + .await + { + Ok(control) => control, + Err(status) => { + if self + .lifecycle_event_fences + .stop_requested(&sandbox.id, &sandbox.name) + { + debug!( + sandbox_id = %sandbox.id, + "Ignoring Docker supervisor startup interruption after an explicit stop" + ); + return span_status.finish(Ok(())); + } + let _ = self + .docker + .remove_container( + &container_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + let _ = remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "ControlSupervisorStartFailed", + status.message(), + )); + } + }; + if self + .lifecycle_event_fences + .stop_requested(&sandbox.id, &sandbox.name) + { + stop_docker_control_process(control).await; + debug!( + sandbox_id = %sandbox.id, + "Discarded Docker supervisor that became ready after an explicit stop" + ); + return span_status.finish(Ok(())); + } + self.replace_control_process(&sandbox.id, control).await; self.publish_docker_progress( &sandbox.id, "Started", format!("Started Docker container \"{container_name}\""), HashMap::from([("container_name".to_string(), container_name)]), ); - if let Err(err) = self - .publish_container_snapshot(&sandbox.id, &sandbox.name) + span_status.finish(Ok(())) + } + + async fn replace_control_process(&self, sandbox_id: &str, process: DockerControlProcess) { + let previous = self + .control_processes + .lock() .await - { - warn!( - sandbox_id = %sandbox.id, - error = %err, - "Failed to publish Docker sandbox snapshot after start" - ); + .insert(sandbox_id.to_string(), process); + if let Some(previous) = previous { + stop_docker_control_process(previous).await; } + } - span_status.finish(Ok(())) + async fn clear_runtime_failure(&self, sandbox_id: &str) { + self.runtime_failures.lock().await.remove(sandbox_id); + } + + fn control_failure_context( + &self, + sandbox: DriverSandbox, + container_id: String, + ) -> DockerRuntimeFailureContext { + DockerRuntimeFailureContext { + docker: self.docker.clone(), + events: self.events.clone(), + failures: self.runtime_failures.clone(), + sandbox, + sandbox_namespace: self.config.sandbox_namespace.clone(), + container_id, + stop_timeout_secs: self.config.stop_timeout_secs, + } + } + + async fn apply_runtime_failure(&self, sandbox: &mut DriverSandbox) { + let container_is_running = sandbox.status.as_ref().is_some_and(|status| { + status.conditions.iter().any(|condition| { + condition.r#type == "Ready" + && condition.status == "True" + && condition.reason == "BackendReady" + }) + }); + if !container_is_running { + return; + } + let failure = self.runtime_failures.lock().await.get(&sandbox.id).cloned(); + if let Some(failure) = failure { + set_sandbox_ready_condition(sandbox, error_condition(failure.reason, &failure.message)); + } + } + + async fn stop_control_process(&self, sandbox_id: &str) { + let process = self.control_processes.lock().await.remove(sandbox_id); + if let Some(process) = process { + stop_docker_control_process(process).await; + } + } + + async fn remove_auxiliary_containers_for_sandbox( + &self, + sandbox_id: &str, + ) -> Result { + let filters = managed_resource_label_filters( + &self.config.sandbox_namespace, + [format!("{LABEL_SANDBOX_ID}={sandbox_id}")], + ); + let containers = self + .docker + .list_containers(Some( + ListContainersOptionsBuilder::default() + .all(true) + .filters(&filters) + .build(), + )) + .await + .map_err(|error| internal_status("list Docker auxiliary containers", error))?; + let mut removed = false; + for container in containers { + let role = container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_ISOLATION_ROLE)) + .map(String::as_str); + if !matches!( + role, + Some( + LABEL_ISOLATION_ROLE_SUPERVISOR + | LABEL_ISOLATION_ROLE_STAGING + | LABEL_ISOLATION_ROLE_IDENTITY + ) + ) { + continue; + } + let Some(target) = summary_container_target(&container) else { + continue; + }; + self.docker + .remove_container( + &target, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await + .or_else(|error| { + if is_not_found_error(&error) || is_removal_in_progress_error(&error) { + Ok(()) + } else { + Err(error) + } + }) + .map_err(|error| internal_status("remove Docker auxiliary container", error))?; + removed = true; + } + Ok(removed) + } + + async fn reconcile_runtime_resources_at_startup(&self) -> Result<(), Status> { + let sandboxes = self.list_managed_container_summaries().await?; + let sandbox_ids = sandboxes + .iter() + .filter_map(|container| { + container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .cloned() + }) + .collect::>(); + + let filters = managed_resource_label_filters(&self.config.sandbox_namespace, []); + let auxiliary = self + .docker + .list_containers(Some( + ListContainersOptionsBuilder::default() + .all(true) + .filters(&filters) + .build(), + )) + .await + .map_err(|error| internal_status("list Docker startup resources", error))?; + for container in auxiliary { + let role = container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_ISOLATION_ROLE)) + .map(String::as_str); + if !matches!( + role, + Some( + LABEL_ISOLATION_ROLE_SUPERVISOR + | LABEL_ISOLATION_ROLE_STAGING + | LABEL_ISOLATION_ROLE_IDENTITY + ) + ) { + continue; + } + let Some(target) = summary_container_target(&container) else { + continue; + }; + self.docker + .remove_container( + &target, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await + .or_else(|error| { + if is_not_found_error(&error) { + Ok(()) + } else { + Err(error) + } + }) + .map_err(|error| internal_status("remove stale Docker auxiliary", error))?; + } + + let volume_filters = label_filters([ + format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}"), + format!( + "{LABEL_SANDBOX_NAMESPACE}={}", + self.config.sandbox_namespace + ), + format!("{LABEL_ISOLATION_TOPOLOGY}={LABEL_ISOLATION_TOPOLOGY_CAPABILITY_FREE}"), + ]); + let volumes = self + .docker + .list_volumes(Some( + ListVolumesOptionsBuilder::default() + .filters(&volume_filters) + .build(), + )) + .await + .map_err(|error| internal_status("list Docker startup volumes", error))?; + for volume in volumes.volumes.unwrap_or_default() { + let sandbox_id = volume.labels.get(LABEL_SANDBOX_ID); + if sandbox_id.is_some_and(|id| sandbox_ids.contains(id)) { + continue; + } + self.docker + .remove_volume( + &volume.name, + None::, + ) + .await + .or_else(|error| { + if is_not_found_error(&error) { + Ok(()) + } else { + Err(error) + } + }) + .map_err(|error| internal_status("remove orphan Docker channel volume", error))?; + } + + for sandbox in &sandboxes { + if sandbox.state == Some(ContainerSummaryStateEnum::RUNNING) + && let Err(error) = self.ensure_control_process_for_container(sandbox).await + { + warn!( + sandbox_id = sandbox + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .map_or("unknown", String::as_str), + %error, + "Failed to restore Docker supervisor during startup reconciliation" + ); + } + } + Ok(()) + } + + async fn ensure_control_process_for_container( + &self, + container: &ContainerSummary, + ) -> Result<(), Status> { + let Some(sandbox) = sandbox_from_container_summary(container) else { + return Err(Status::internal( + "managed Docker container is missing sandbox identity labels", + )); + }; + let stale = { + let mut processes = self.control_processes.lock().await; + match processes.get(&sandbox.id) { + Some(process) if !process.task.is_finished() => return Ok(()), + Some(_) => processes.remove(&sandbox.id), + None => None, + } + }; + if let Some(stale) = stale { + stop_docker_control_process(stale).await; + } + let Some(topology) = read_docker_boundary_topology(&sandbox.id, &self.config).await? else { + let container_id = summary_container_target(container) + .ok_or_else(|| Status::internal("managed Docker container has no id or name"))?; + let failure_context = self.control_failure_context(sandbox.clone(), container_id); + let status = Status::failed_precondition( + "Docker sandbox topology is missing; refusing to leave the workload running without its supervisor", + ); + handle_docker_runtime_failure( + failure_context, + "ControlSupervisorExited", + status.message().to_string(), + ) + .await; + return Err(status); + }; + let container_id = summary_container_target(container) + .ok_or_else(|| Status::internal("managed Docker container has no id or name"))?; + self.clear_runtime_failure(&sandbox.id).await; + let failure_context = self.control_failure_context(sandbox.clone(), container_id); + let process = match spawn_docker_control_process( + &self.docker, + &sandbox, + &self.config, + &topology, + failure_context.clone(), + ) + .await + { + Ok(process) => process, + Err(status) => { + handle_docker_runtime_failure( + failure_context, + "ControlSupervisorExited", + format!( + "failed to start Docker control supervisor: {}", + status.message() + ), + ) + .await; + return Err(status); + } + }; + self.replace_control_process(&sandbox.id, process).await; + Ok(()) } async fn delete_sandbox_inner( @@ -1129,14 +1939,17 @@ impl DockerComputeDriver { sandbox_id: &str, sandbox_name: &str, ) -> Result { - let pending = self - .remove_pending_sandbox(sandbox_id, sandbox_name) - .await?; + let pending = self.remove_pending_sandbox(sandbox_id, sandbox_name).await; if let Some(record) = pending.as_ref() && let Some(task) = record.task.as_ref() { task.abort(); } + if let Some(record) = pending.as_ref() { + self.stop_control_process(&record.sandbox.id).await; + self.remove_auxiliary_containers_for_sandbox(&record.sandbox.id) + .await?; + } let Some(container) = self .find_managed_container_summary(sandbox_id, sandbox_name) @@ -1153,11 +1966,25 @@ impl DockerComputeDriver { .await { Ok(()) => { - cleanup_sandbox_token_file(&record.sandbox, &self.config); + self.clear_runtime_failure(&record.sandbox.id).await; + remove_docker_channel_volume_by_id( + &self.docker, + &record.sandbox.id, + &self.config, + ) + .await?; + cleanup_docker_boundary_state(&record.sandbox, &self.config); return Ok(true); } Err(err) if is_not_found_error(&err) => { - cleanup_sandbox_token_file(&record.sandbox, &self.config); + self.clear_runtime_failure(&record.sandbox.id).await; + let _ = remove_docker_channel_volume_by_id( + &self.docker, + &record.sandbox.id, + &self.config, + ) + .await; + cleanup_docker_boundary_state(&record.sandbox, &self.config); return Ok(true); } Err(err) => { @@ -1165,16 +1992,29 @@ impl DockerComputeDriver { } } } - // Container gone and no in-memory record survived (gateway - // restarted after an out-of-band `docker rm`). DeleteSandbox is - // the only thing that ever reclaims the token file, so reclaim it - // here too. - cleanup_sandbox_token_file_for_delete(sandbox_id, None, &self.config); + if !sandbox_id.is_empty() { + self.stop_control_process(sandbox_id).await; + let removed = self + .remove_auxiliary_containers_for_sandbox(sandbox_id) + .await?; + remove_docker_channel_volume_by_id(&self.docker, sandbox_id, &self.config).await?; + cleanup_docker_boundary_state_by_id(sandbox_id, &self.config); + self.clear_runtime_failure(sandbox_id).await; + return Ok(removed); + } return Ok(false); }; let Some(target) = summary_container_target(&container) else { return Ok(pending.is_some()); }; + let resolved_sandbox_id = container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .map_or(sandbox_id, String::as_str); + self.stop_control_process(resolved_sandbox_id).await; + self.remove_auxiliary_containers_for_sandbox(resolved_sandbox_id) + .await?; match self .docker @@ -1185,11 +2025,21 @@ impl DockerComputeDriver { .await { Ok(()) => { - cleanup_sandbox_token_file_for_delete(sandbox_id, pending.as_ref(), &self.config); + self.clear_runtime_failure(resolved_sandbox_id).await; + remove_docker_channel_volume_by_id(&self.docker, resolved_sandbox_id, &self.config) + .await?; + cleanup_docker_boundary_state_by_id(resolved_sandbox_id, &self.config); Ok(true) } Err(err) if is_not_found_error(&err) => { - cleanup_sandbox_token_file_for_delete(sandbox_id, pending.as_ref(), &self.config); + self.clear_runtime_failure(resolved_sandbox_id).await; + let _ = remove_docker_channel_volume_by_id( + &self.docker, + resolved_sandbox_id, + &self.config, + ) + .await; + cleanup_docker_boundary_state_by_id(resolved_sandbox_id, &self.config); Ok(pending.is_some()) } Err(err) => Err(internal_status("delete docker sandbox container", err)), @@ -1201,14 +2051,17 @@ impl DockerComputeDriver { .find_managed_container_summary(sandbox_id, sandbox_name) .await? else { - if let Some(record) = self - .remove_pending_sandbox(sandbox_id, sandbox_name) - .await? - { + if let Some(record) = self.remove_pending_sandbox(sandbox_id, sandbox_name).await { + self.stop_control_process(&record.sandbox.id).await; + self.remove_auxiliary_containers_for_sandbox(&record.sandbox.id) + .await?; + self.clear_runtime_failure(&record.sandbox.id).await; if let Some(task) = record.task { task.abort(); } - cleanup_sandbox_token_file(&record.sandbox, &self.config); + remove_docker_channel_volume_by_id(&self.docker, &record.sandbox.id, &self.config) + .await?; + cleanup_docker_boundary_state(&record.sandbox, &self.config); self.publish_deleted(record.sandbox.id); return Ok(()); } @@ -1217,8 +2070,16 @@ impl DockerComputeDriver { let Some(target) = summary_container_target(&container) else { return Err(Status::not_found("sandbox container has no id or name")); }; + let resolved_sandbox_id = container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .map_or(sandbox_id, String::as_str); + self.stop_control_process(resolved_sandbox_id).await; + self.remove_auxiliary_containers_for_sandbox(resolved_sandbox_id) + .await?; - match self + let result = match self .docker .stop_container( &target, @@ -1234,7 +2095,11 @@ impl DockerComputeDriver { Err(err) if is_not_modified_error(&err) => Ok(()), Err(err) if is_not_found_error(&err) => Err(Status::not_found("sandbox not found")), Err(err) => Err(internal_status("stop docker sandbox container", err)), + }; + if result.is_ok() { + self.clear_runtime_failure(resolved_sandbox_id).await; } + result } /// Start a managed sandbox container that was previously stopped. Used @@ -1262,10 +2127,11 @@ impl DockerComputeDriver { ) -> Result { let span_status = openshell_otel::ErrorStatusGuard::current(); require_sandbox_identifier(sandbox_id, sandbox_name)?; + self.lifecycle_event_fences + .clear_stop(sandbox_id, sandbox_name); self.lifecycle_event_fences.begin_start(sandbox_id); - let result = self - .start_sandbox_with_lifecycle_fence(sandbox_id, sandbox_name) - .await; + let result = + Box::pin(self.start_sandbox_with_lifecycle_fence(sandbox_id, sandbox_name)).await; self.lifecycle_event_fences.finish_start(sandbox_id); span_status.finish(result) } @@ -1284,20 +2150,14 @@ impl DockerComputeDriver { let Some(target) = summary_container_target(&container) else { return Ok(false); }; + let inspected = self + .docker + .inspect_container(&target, None) + .await + .map_err(|error| internal_status("inspect Docker sandbox outer fence", error))?; + validate_docker_outer_fence(&inspected)?; let state = container.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); - if !container_state_needs_start(state) { - return Ok(true); - } - - // Fence a poll that observed this stopped run but has not published it - // yet. Use Docker's transition timestamp so a later, genuine exit from - // the restarted container remains observable. let previous_finished_at = if state == ContainerSummaryStateEnum::EXITED { - let inspected = self - .docker - .inspect_container(&target, None) - .await - .map_err(|err| internal_status("inspect docker sandbox before start", err))?; inspected .state .as_ref() @@ -1306,26 +2166,110 @@ impl DockerComputeDriver { } else { None }; + drop(inspected); + if !container_state_needs_start(state) { + self.ensure_control_process_for_container(&container) + .await?; + return Ok(true); + } + + // Fence a poll that observed this stopped run but has not published it + // yet. Use Docker's transition timestamp so a later, genuine exit from + // the restarted container remains observable. self.lifecycle_event_fences .record_previous_exit(sandbox_id, previous_finished_at.as_deref()); + let resolved_sandbox_id = container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .map_or(sandbox_id, String::as_str); + let Some(topology) = + read_docker_boundary_topology(resolved_sandbox_id, &self.config).await? + else { + return Err(Status::failed_precondition( + "Docker sandbox topology is missing; refusing to start the workload without its supervisor", + )); + }; + let boundary_config = tokio::fs::read( + docker_boundary_state_dir_by_id(resolved_sandbox_id, &self.config)? + .join(BOUNDARY_CONFIG_FILE), + ) + .await + .map_err(|error| { + Status::failed_precondition(format!( + "read Docker sandbox bootstrap for restart: {error}" + )) + })?; + let boundary_directory = + docker_boundary_state_dir_by_id(resolved_sandbox_id, &self.config)?; + let boundary_certificate = + tokio::fs::read(boundary_directory.join(BOUNDARY_CERTIFICATE_FILE)) + .await + .map_err(|error| { + Status::failed_precondition(format!( + "read Docker sandbox channel certificate for restart: {error}" + )) + })?; + let boundary_private_key = + tokio::fs::read(boundary_directory.join(BOUNDARY_PRIVATE_KEY_FILE)) + .await + .map_err(|error| { + Status::failed_precondition(format!( + "read Docker sandbox channel private key for restart: {error}" + )) + })?; + let boundary_client_ca = tokio::fs::read(boundary_directory.join(BOUNDARY_CLIENT_CA_FILE)) + .await + .map_err(|error| { + Status::failed_precondition(format!( + "read Docker sandbox channel client CA for restart: {error}" + )) + })?; + let workspace_root = tokio::fs::read_to_string( + docker_boundary_state_dir_by_id(resolved_sandbox_id, &self.config)? + .join(WORKSPACE_ROOT_FILE), + ) + .await + .map_err(|error| { + Status::failed_precondition(format!( + "read Docker sandbox workspace for restart: {error}" + )) + })?; + stage_docker_sandbox_bundle( + &self.docker, + &target, + &self.config, + &topology.workload_identity, + &boundary_config, + DockerSandboxTls { + certificate: &boundary_certificate, + private_key: &boundary_private_key, + client_ca: &boundary_client_ca, + }, + &workspace_root, + ) + .await?; + match self.docker.start_container(&target, None).await { - Ok(()) => Ok(true), + Ok(()) => {} // Already running — race with another start path or the // restart policy. Treat as success. - Err(err) if is_not_modified_error(&err) => Ok(true), - Err(err) if is_not_found_error(&err) => Ok(false), - Err(err) => Err(internal_status("start docker sandbox container", err)), + Err(err) if is_not_modified_error(&err) => {} + Err(err) if is_not_found_error(&err) => return Ok(false), + Err(err) => return Err(internal_status("start docker sandbox container", err)), } + self.ensure_control_process_for_container(&container) + .await?; + Ok(true) } async fn reserve_pending_sandbox(&self, sandbox: &DriverSandbox) -> Result<(), Status> { let mut pending = self.pending.lock().await; - if pending.values().any(|record| { - record.sandbox.id == sandbox.id - || (record.sandbox.name == sandbox.name - && record.sandbox.workspace == sandbox.workspace) - }) { + if pending + .values() + .any(|record| record.sandbox.id == sandbox.id || record.sandbox.name == sandbox.name) + { return Err(Status::already_exists("sandbox already exists")); } @@ -1334,7 +2278,7 @@ impl DockerComputeDriver { PendingSandboxRecord { sandbox: pending_sandbox_snapshot( sandbox, - &self.config.sandbox_label, + &self.config.sandbox_namespace, provisioning_condition(), false, ), @@ -1348,12 +2292,12 @@ impl DockerComputeDriver { &self, sandbox_id: &str, sandbox_name: &str, - ) -> Result, Status> { + ) -> Option { let pending = self.pending.lock().await; - let Some(id) = resolve_pending_id(&pending, sandbox_id, sandbox_name)? else { - return Ok(None); - }; - Ok(pending.get(&id).map(|record| record.sandbox.clone())) + pending + .values() + .find(|record| pending_sandbox_matches(&record.sandbox, sandbox_id, sandbox_name)) + .map(|record| record.sandbox.clone()) } async fn pending_snapshot_map(&self) -> HashMap { @@ -1373,12 +2317,12 @@ impl DockerComputeDriver { &self, sandbox_id: &str, sandbox_name: &str, - ) -> Result, Status> { + ) -> Option { let mut pending = self.pending.lock().await; - let Some(id) = resolve_pending_id(&pending, sandbox_id, sandbox_name)? else { - return Ok(None); - }; - Ok(pending.remove(&id)) + let id = pending.iter().find_map(|(id, record)| { + pending_sandbox_matches(&record.sandbox, sandbox_id, sandbox_name).then(|| id.clone()) + })?; + pending.remove(&id) } async fn fail_pending_sandbox( @@ -1386,10 +2330,10 @@ impl DockerComputeDriver { sandbox: &DriverSandbox, failure: &DockerProvisioningFailure, ) { - cleanup_sandbox_token_file(sandbox, &self.config); + cleanup_docker_boundary_state(sandbox, &self.config); let snapshot = pending_sandbox_snapshot( sandbox, - &self.config.sandbox_label, + &self.config.sandbox_namespace, error_condition(failure.reason, &failure.message), false, ); @@ -1420,11 +2364,16 @@ impl DockerComputeDriver { sandbox_id: &str, sandbox_name: &str, ) -> Result<(), Status> { + if let Some(pending) = self.pending_snapshot(sandbox_id, sandbox_name).await { + self.publish_sandbox_snapshot(pending); + return Ok(()); + } if let Some(summary) = self .find_managed_container_summary(sandbox_id, sandbox_name) .await? - && let Some(sandbox) = sandbox_from_container_summary(&summary) + && let Some(mut sandbox) = sandbox_from_container_summary(&summary) { + self.apply_runtime_failure(&mut sandbox).await; self.publish_sandbox_snapshot(sandbox); } Ok(()) @@ -1595,7 +2544,7 @@ impl DockerComputeDriver { } async fn list_managed_container_summaries(&self) -> Result, Status> { - let filters = managed_container_label_filters(&self.config.sandbox_label, []); + let filters = managed_container_label_filters(&self.config.sandbox_namespace, []); self.docker .list_containers(Some( ListContainersOptionsBuilder::default() @@ -1620,7 +2569,7 @@ impl DockerComputeDriver { } let filters = - managed_container_label_filters(&self.config.sandbox_label, label_filter_values); + managed_container_label_filters(&self.config.sandbox_namespace, label_filter_values); let containers = self .docker .list_containers(Some( @@ -1633,14 +2582,21 @@ impl DockerComputeDriver { .map_err(|err| internal_status("find Docker sandbox container", err))?; Ok(containers.into_iter().find(|summary| { - summary.labels.as_ref().is_some_and(|labels| { - managed_container_identity_matches( - labels, - &self.config.sandbox_label, - sandbox_id, - sandbox_name, - ) - }) + let Some(labels) = summary.labels.as_ref() else { + return false; + }; + let namespace_matches = labels + .get(LABEL_SANDBOX_NAMESPACE) + .is_some_and(|value| value == &self.config.sandbox_namespace); + let id_matches = sandbox_id.is_empty() + || labels + .get(LABEL_SANDBOX_ID) + .is_some_and(|value| value == sandbox_id); + let name_matches = sandbox_name.is_empty() + || labels + .get(LABEL_SANDBOX_NAME) + .is_some_and(|value| value == sandbox_name); + namespace_matches && id_matches && name_matches })) } @@ -1649,8 +2605,9 @@ impl DockerComputeDriver { sandbox_id: &str, image: &str, ) -> Result { - let inspect = match self.config.image_pull_policy { - ImagePullPolicy::IfNotPresent => { + let policy = self.config.image_pull_policy.trim().to_ascii_lowercase(); + let inspect = match policy.as_str() { + "" | "ifnotpresent" => { if let Ok(inspect) = self.docker.inspect_image(image).await { self.publish_docker_progress( sandbox_id, @@ -1667,14 +2624,14 @@ impl DockerComputeDriver { .map_err(|err| internal_status("inspect Docker image after pull", err))? } } - ImagePullPolicy::Always => { + "always" => { self.pull_image(sandbox_id, image).await?; self.docker .inspect_image(image) .await .map_err(|err| internal_status("inspect Docker image after pull", err))? } - ImagePullPolicy::Never => match self.docker.inspect_image(image).await { + "never" => match self.docker.inspect_image(image).await { Ok(inspect) => { self.publish_docker_progress( sandbox_id, @@ -1686,15 +2643,15 @@ impl DockerComputeDriver { } Err(err) if is_not_found_error(&err) => { return Err(Status::failed_precondition(format!( - "docker image '{image}' is not present locally and image_pull_policy = \"never\"" + "docker image '{image}' is not present locally and image_pull_policy=Never" ))); } Err(err) => return Err(internal_status("inspect Docker image", err)), }, - ImagePullPolicy::Newer => { - return Err(Status::failed_precondition( - "image_pull_policy = \"newer\" is supported only by the Podman compute driver", - )); + other => { + return Err(Status::failed_precondition(format!( + "unsupported docker image_pull_policy '{other}'; expected Always, IfNotPresent, or Never", + ))); } }; @@ -1764,8 +2721,38 @@ impl DockerComputeDriver { // Standalone and in-process servers both use this wrapper. Delegating to the // driver's canonical tonic implementation keeps request validation and Docker // operation spans identical across both deployment modes. -#[tonic::async_trait] -impl ComputeDriver for ComputeDriverService { +fn validate_docker_outer_fence( + inspected: &bollard::models::ContainerInspectResponse, +) -> Result<(), Status> { + let network_mode = inspected + .host_config + .as_ref() + .and_then(|config| config.network_mode.as_deref()); + if network_mode != Some("none") { + return Err(Status::failed_precondition(format!( + "Docker sandbox outer fence requires network_mode=none, got {}", + network_mode.unwrap_or("") + ))); + } + let unexpected_networks = inspected + .network_settings + .as_ref() + .and_then(|settings| settings.networks.as_ref()) + .into_iter() + .flat_map(HashMap::keys) + .filter(|network| network.as_str() != "none") + .cloned() + .collect::>(); + if !unexpected_networks.is_empty() { + return Err(Status::failed_precondition(format!( + "Docker sandbox outer fence found attached networks: {}", + unexpected_networks.join(", ") + ))); + } + Ok(()) +} +#[tonic::async_trait] +impl ComputeDriver for ComputeDriverService { type WatchSandboxesStream = WatchStream; async fn authenticate_sandbox( @@ -2066,10 +3053,24 @@ impl ComputeDriver for DockerComputeDriver { let request = request.into_inner(); require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; - self.stop_sandbox_inner(&request.sandbox_id, &request.sandbox_name) - .await?; - self.publish_container_snapshot(&request.sandbox_id, &request.sandbox_name) - .await?; + self.lifecycle_event_fences + .request_stop(&request.sandbox_id, &request.sandbox_name); + if let Err(error) = self + .stop_sandbox_inner(&request.sandbox_id, &request.sandbox_name) + .await + { + self.lifecycle_event_fences + .clear_stop(&request.sandbox_id, &request.sandbox_name); + return Err(error); + } + if let Err(error) = self + .publish_container_snapshot(&request.sandbox_id, &request.sandbox_name) + .await + { + self.lifecycle_event_fences + .clear_stop(&request.sandbox_id, &request.sandbox_name); + return Err(error); + } span_status.finish(Ok(Response::new(StopSandboxResponse {}))) } @@ -2078,7 +3079,13 @@ impl ComputeDriver for DockerComputeDriver { request: Request, ) -> Result, Status> { let request = request.into_inner(); - if !Self::start_sandbox(self, &request.sandbox_id, &request.sandbox_name).await? { + if !Box::pin(Self::start_sandbox( + self, + &request.sandbox_id, + &request.sandbox_name, + )) + .await? + { return Err(Status::not_found("sandbox not found")); } self.publish_container_snapshot(&request.sandbox_id, &request.sandbox_name) @@ -2108,7 +3115,8 @@ impl ComputeDriver for DockerComputeDriver { let deleted = self .delete_sandbox_inner(&request.sandbox_id, &request.sandbox_name) .await?; - self.lifecycle_event_fences.remove(&event_sandbox_id); + self.lifecycle_event_fences + .remove(&event_sandbox_id, &request.sandbox_name); if deleted && !event_sandbox_id.is_empty() { let _ = self.events.send(WatchSandboxesEvent { payload: Some(watch_sandboxes_event::Payload::Deleted( @@ -2228,73 +3236,9 @@ fn pending_sandbox_snapshot( } } -/// Decides whether a managed container satisfies a lifecycle request. -/// -/// `sandbox_id` is authoritative, matching [`resolve_pending_id`]. Requiring -/// the name to agree as well would discard a correct id match whenever the -/// caller pairs it with a stale name, leaving the container and its token file -/// behind while the driver reports the sandbox as absent. -/// -/// A request with no identifier matches nothing. `require_sandbox_identifier` -/// rejects that upstream, but the label filters degenerate to "every managed -/// container in the namespace", so this does not rely on the caller to guard it. -fn managed_container_identity_matches( - labels: &HashMap, - namespace: &str, - sandbox_id: &str, - sandbox_name: &str, -) -> bool { - if labels - .get(LABEL_SANDBOX_NAMESPACE) - .is_none_or(|value| value != namespace) - { - return false; - } - if !sandbox_id.is_empty() { - return labels - .get(LABEL_SANDBOX_ID) - .is_some_and(|value| value == sandbox_id); - } - !sandbox_name.is_empty() - && labels - .get(LABEL_SANDBOX_NAME) - .is_some_and(|value| value == sandbox_name) -} - -/// Resolves a lifecycle request to at most one pending sandbox id. -/// -/// `sandbox_id` is authoritative: when the caller supplies one, the name is -/// never consulted as an alternative. The name fallback rejects ambiguity -/// instead of letting `HashMap` iteration order pick a match, because sandbox -/// names are unique per workspace and the driver request carries no workspace. -fn resolve_pending_id( - pending: &HashMap, - sandbox_id: &str, - sandbox_name: &str, -) -> Result, Status> { - if !sandbox_id.is_empty() { - return Ok(pending - .contains_key(sandbox_id) - .then(|| sandbox_id.to_string())); - } - if sandbox_name.is_empty() { - return Ok(None); - } - - let mut matches = pending - .iter() - .filter(|(_, record)| record.sandbox.name == sandbox_name) - .map(|(id, _)| id.clone()); - - let Some(id) = matches.next() else { - return Ok(None); - }; - if matches.next().is_some() { - return Err(Status::failed_precondition( - "sandbox_name matches multiple pending sandboxes; specify sandbox_id", - )); - } - Ok(Some(id)) +fn pending_sandbox_matches(sandbox: &DriverSandbox, sandbox_id: &str, sandbox_name: &str) -> bool { + (!sandbox_id.is_empty() && sandbox.id == sandbox_id) + || (!sandbox_name.is_empty() && sandbox.name == sandbox_name) } fn provisioning_condition() -> DriverCondition { @@ -2317,6 +3261,21 @@ fn error_condition(reason: &str, message: &str) -> DriverCondition { } } +fn set_sandbox_ready_condition(sandbox: &mut DriverSandbox, condition: DriverCondition) { + let Some(status) = sandbox.status.as_mut() else { + return; + }; + if let Some(existing) = status + .conditions + .iter_mut() + .find(|existing| existing.r#type == "Ready") + { + *existing = condition; + } else { + status.conditions.push(condition); + } +} + fn platform_event( source: &str, event_type: &str, @@ -2738,96 +3697,114 @@ fn docker_volume_is_bind_backed(volume: &bollard::models::Volume) -> bool { }) } -/// Verify the configured credential without exposing its contents. Docker -/// bind-mounts the root-owned file directly, unlike Podman which uses a native -/// secret object; this preflight makes a bad file fail before any sandbox is -/// created. -fn validate_docker_proxy_auth_file(config: &UpstreamProxyConfig) -> CoreResult<()> { - let Some(path) = config.proxy_auth_file.as_ref() else { - return Ok(()); - }; - let raw = openshell_core::driver_utils::read_upstream_proxy_credential_file( - path.to_str() - .ok_or_else(|| Error::config("proxy_auth_file must be valid UTF-8"))?, - ) - .map_err(Error::config)?; - openshell_core::driver_utils::parse_upstream_proxy_credential(&raw) - .map_err(|error| Error::config(format!("proxy_auth_file is invalid: {error}")))?; - Ok(()) +fn build_binds(_sandbox: &DriverSandbox, _config: &DockerDriverRuntimeConfig) -> Vec { + Vec::new() } -/// Build immutable operator-owned proxy arguments. Credentials never appear on -/// argv: only the fixed in-container root-only file path is supplied. -fn docker_upstream_proxy_cli_args(config: &UpstreamProxyConfig) -> Vec { - let mut args = Vec::new(); - if let Some(url) = config.https_proxy.as_ref() { - args.extend(["--upstream-proxy".to_string(), url.clone()]); - } - if let Some(no_proxy) = config.no_proxy.as_ref() { - args.extend(["--upstream-no-proxy".to_string(), no_proxy.clone()]); - } - if config.proxy_auth_file.is_some() { - args.extend([ - "--upstream-proxy-auth-file".to_string(), - UPSTREAM_PROXY_AUTH_MOUNT_PATH.to_string(), - ]); - } - if config.proxy_auth_allow_insecure == Some(true) { - args.push("--upstream-proxy-auth-allow-insecure".to_string()); - } - if config.proxy_connect_by_hostname == Some(true) { - args.push("--upstream-proxy-connect-by-hostname".to_string()); - } - args +fn docker_boundary_state_dir( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result { + docker_boundary_state_dir_by_id(&sandbox.id, config) +} + +fn docker_boundary_state_dir_by_id( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> Result { + sandbox_token_host_path_by_id(sandbox_id, config).and_then(|path| { + path.parent() + .map(Path::to_path_buf) + .ok_or_else(|| Status::internal("docker boundary state path has no parent")) + }) } -fn build_binds( +fn docker_channel_volume_name( sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, -) -> Result, Status> { - let mut binds = vec![format!( - "{}:{}:ro,z", - config.supervisor_bin.display(), - SUPERVISOR_MOUNT_PATH - )]; - if let Some(tls) = &config.guest_tls { - binds.push(format!("{}:{}:ro,z", tls.ca.display(), TLS_CA_MOUNT_PATH)); - binds.push(format!( - "{}:{}:ro,z", - tls.cert.display(), - TLS_CERT_MOUNT_PATH - )); - binds.push(format!("{}:{}:ro,z", tls.key.display(), TLS_KEY_MOUNT_PATH)); - } - if sandbox - .spec - .as_ref() - .is_some_and(|spec| !spec.sandbox_token.is_empty()) - { - binds.push(format!( - "{}:{}:ro,z", - sandbox_token_host_path(sandbox, config)?.display(), - SANDBOX_TOKEN_MOUNT_PATH - )); - } - if let Some(path) = config.upstream_proxy.proxy_auth_file.as_ref() { - binds.push(format!( - "{}:{}:ro,z", - path.display(), - UPSTREAM_PROXY_AUTH_MOUNT_PATH - )); - } - if let Some(socket) = config.provider_spiffe_workload_api_socket.as_ref() { - let parent = socket.parent().ok_or_else(|| { - Status::failed_precondition("provider SPIFFE socket has no parent directory") +) -> String { + docker_channel_volume_name_by_id(&sandbox.id, config) +} + +fn docker_channel_volume_name_by_id( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> String { + let mut hasher = Sha256::new(); + hasher.update(config.sandbox_namespace.as_bytes()); + hasher.update([0]); + hasher.update(sandbox_id.as_bytes()); + let digest = format!("{:x}", hasher.finalize()); + format!("openshell-channel-{}", &digest[..32]) +} + +async fn create_docker_channel_volume( + docker: &Docker, + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result<(), Status> { + let name = docker_channel_volume_name(sandbox, config); + let expected_labels = HashMap::from([ + ( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ), + (LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()), + ( + LABEL_SANDBOX_NAMESPACE.to_string(), + config.sandbox_namespace.clone(), + ), + ( + LABEL_ISOLATION_TOPOLOGY.to_string(), + LABEL_ISOLATION_TOPOLOGY_CAPABILITY_FREE.to_string(), + ), + ]); + docker + .create_volume(VolumeCreateRequest { + name: Some(name.clone()), + labels: Some(expected_labels.clone()), + ..Default::default() + }) + .await + .map_err(|error| { + Status::internal(format!("create Docker sandbox channel volume: {error}")) })?; - binds.push(format!( - "{}:{}:ro", - parent.display(), - PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR - )); + let volume = docker.inspect_volume(&name).await.map_err(|error| { + Status::internal(format!("inspect Docker sandbox channel volume: {error}")) + })?; + if volume.driver != "local" + || !volume.options.is_empty() + || expected_labels + .iter() + .any(|(key, value)| volume.labels.get(key) != Some(value)) + { + return Err(Status::failed_precondition(format!( + "Docker sandbox channel volume '{name}' already exists without the expected local-driver ownership labels" + ))); } - Ok(binds) + Ok(()) +} + +async fn remove_docker_channel_volume_by_id( + docker: &Docker, + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> Result<(), Status> { + let name = docker_channel_volume_name_by_id(sandbox_id, config); + docker + .remove_volume( + &name, + None::, + ) + .await + .or_else(|error| { + if is_not_found_error(&error) { + Ok(()) + } else { + Err(error) + } + }) + .map_err(|error| Status::internal(format!("remove Docker sandbox channel volume: {error}"))) } fn sandbox_token_host_path( @@ -2843,7 +3820,7 @@ fn sandbox_token_host_path_by_id( ) -> Result { openshell_core::driver_utils::sandbox_token_path( "docker-sandbox-tokens", - Some(&config.sandbox_label), + Some(&config.sandbox_namespace), sandbox_id, ) .map_err(|err| { @@ -2889,173 +3866,1032 @@ async fn write_sandbox_token_file( Ok(true) } -fn cleanup_sandbox_token_file(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) { - cleanup_sandbox_token_file_by_id(&sandbox.id, config); +fn prepare_docker_boundary_state_dir( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result { + let directory = docker_boundary_state_dir(sandbox, config)?; + openshell_core::paths::create_dir_restricted(&directory).map_err(|error| { + Status::internal(format!( + "create Docker boundary state directory {}: {error}", + directory.display() + )) + })?; + Ok(directory) +} + +async fn write_docker_boundary_file(path: &Path, contents: &[u8]) -> Result<(), Status> { + tokio::fs::write(path, contents).await.map_err(|error| { + Status::internal(format!( + "write Docker boundary file {}: {error}", + path.display() + )) + })?; + openshell_core::paths::set_file_owner_only(path).map_err(|error| { + Status::internal(format!( + "restrict Docker boundary file {}: {error}", + path.display() + )) + }) +} + +fn append_docker_archive_directory( + archive: &mut tar::Builder>, + path: &str, + mode: u32, + uid: u32, + gid: u32, +) -> Result<(), Status> { + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Directory); + header.set_mode(mode); + header.set_uid(u64::from(uid)); + header.set_gid(u64::from(gid)); + header.set_mtime(0); + header.set_size(0); + header.set_cksum(); + archive + .append_data(&mut header, path, std::io::empty()) + .map_err(|error| Status::internal(format!("build Docker sandbox archive: {error}"))) +} + +fn append_docker_archive_file( + archive: &mut tar::Builder>, + path: &str, + mode: u32, + uid: u32, + gid: u32, + contents: &[u8], +) -> Result<(), Status> { + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_mode(mode); + header.set_uid(u64::from(uid)); + header.set_gid(u64::from(gid)); + header.set_mtime(0); + header.set_size(contents.len() as u64); + header.set_cksum(); + archive + .append_data(&mut header, path, contents) + .map_err(|error| Status::internal(format!("build Docker sandbox archive: {error}"))) +} + +#[derive(Clone, Copy)] +struct DockerSandboxTls<'a> { + certificate: &'a [u8], + private_key: &'a [u8], + client_ca: &'a [u8], +} + +fn docker_sandbox_bundle_archive( + sandbox_binary: &[u8], + boundary_config: &[u8], + boundary_tls: DockerSandboxTls<'_>, + identity: &ResolvedWorkloadIdentity, + workspace_root: &str, +) -> Result, Status> { + let mut archive = tar::Builder::new(Vec::new()); + append_docker_archive_directory(&mut archive, ".openshell", 0o755, 0, 0)?; + append_docker_archive_directory(&mut archive, ".openshell/runtime", 0o555, 0, 0)?; + append_docker_archive_directory(&mut archive, ".openshell/channel", 0o755, 0, 0)?; + append_docker_archive_directory( + &mut archive, + ".openshell/channel/sandbox", + // The sandbox owns this directory so it can consume bootstrap files + // and create the control socket. The separate non-root supervisor + // needs execute-only traversal to that known socket path; mutual TLS + // authenticates the endpoint and the files beneath remain 0600. + 0o711, + identity.uid, + identity.gid, + )?; + append_docker_archive_file( + &mut archive, + ".openshell/runtime/openshell-sandbox", + 0o555, + 0, + 0, + sandbox_binary, + )?; + append_docker_archive_file( + &mut archive, + ".openshell/channel/sandbox/bootstrap.json", + 0o600, + identity.uid, + identity.gid, + boundary_config, + )?; + for (path, contents) in [ + ( + ".openshell/channel/sandbox/server.crt", + boundary_tls.certificate, + ), + ( + ".openshell/channel/sandbox/server.key", + boundary_tls.private_key, + ), + ( + ".openshell/channel/sandbox/client-ca.crt", + boundary_tls.client_ca, + ), + ] { + append_docker_archive_file( + &mut archive, + path, + 0o600, + identity.uid, + identity.gid, + contents, + )?; + } + if workspace_root == driver_mounts::DEFAULT_WORKSPACE_ROOT { + // The default workspace is driver-managed. Create it before the + // capability-free sandbox starts because that process deliberately + // has no authority to create or chown a directory beneath `/`. + append_docker_archive_directory( + &mut archive, + workspace_root.trim_start_matches('/'), + 0o700, + identity.uid, + identity.gid, + )?; + } + archive + .into_inner() + .map_err(|error| Status::internal(format!("finish Docker sandbox archive: {error}"))) +} + +async fn stage_docker_sandbox_bundle( + docker: &Docker, + container_id: &str, + config: &DockerDriverRuntimeConfig, + identity: &ResolvedWorkloadIdentity, + boundary_config: &[u8], + boundary_tls: DockerSandboxTls<'_>, + workspace_root: &str, +) -> Result<(), Status> { + let archive = docker_sandbox_bundle_archive( + config.sandbox_binary.as_slice(), + boundary_config, + boundary_tls, + identity, + workspace_root, + )?; + let options = UploadToContainerOptionsBuilder::default() + .path("/") + .copy_uidgid("true") + .build(); + docker + .upload_to_container( + container_id, + Some(options), + bollard::body_full(Bytes::from(archive)), + ) + .await + .map_err(|error| Status::internal(format!("stage Docker sandbox bundle: {error}"))) +} + +async fn prepare_docker_boundary_files( + docker: &Docker, + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, + container_id: &str, + image: &DockerImageMetadata, + workload_identity: &ResolvedWorkloadIdentity, +) -> Result { + let directory = docker_boundary_state_dir(sandbox, config)?; + let workspace_root = driver_mounts::resolve_oci_workspace_root(&image.working_dir) + .map_err(Status::failed_precondition)?; + let bootstrap_token = random_boundary_token(); + let host_gateway_ip = Some(match config.gateway_route { + DockerGatewayRoute::Bridge { bind_address, .. } => bind_address.ip(), + DockerGatewayRoute::HostGateway => IpAddr::V4(Ipv4Addr::LOCALHOST), + }); + let tls = generate_boundary_mutual_tls_material() + .map_err(|error| Status::internal(format!("generate Docker boundary TLS: {error}")))?; + let provisioning = isolation::DockerBoundarySpec { + boundary_id: sandbox.id.clone(), + bootstrap_token, + generation: random_boundary_token(), + session_epoch: random_boundary_token(), + container_id: container_id.to_string(), + image_identity: image.id.clone(), + listener_socket: PathBuf::from(BOUNDARY_SOCKET_MOUNT_PATH), + control_socket: PathBuf::from(BOUNDARY_SOCKET_MOUNT_PATH), + sandbox_tls: BoundaryServerTls { + certificate_chain_path: PathBuf::from(BOUNDARY_CERTIFICATE_MOUNT_PATH), + private_key_path: PathBuf::from(BOUNDARY_PRIVATE_KEY_MOUNT_PATH), + client_ca_certificate_path: PathBuf::from(BOUNDARY_CLIENT_CA_MOUNT_PATH), + }, + supervisor_tls: BoundaryClientTls { + server_name: tls.server_name.clone(), + ca_certificate_pem: tls.ca_certificate_pem.clone(), + certificate_chain_pem: tls.supervisor_certificate_pem.clone(), + private_key_pem: tls.supervisor_private_key_pem.clone(), + }, + host_gateway_ip, + workload_identity: workload_identity.clone(), + child_env: docker_child_environment(sandbox), + } + .provision(); + let boundary_config = provisioning + .boundary_config + .encode() + .map_err(|error| Status::internal(error.to_string()))?; + write_docker_boundary_file(&directory.join(BOUNDARY_CONFIG_FILE), &boundary_config).await?; + write_docker_boundary_file( + &directory.join(BOUNDARY_CERTIFICATE_FILE), + tls.sandbox_certificate_pem.as_bytes(), + ) + .await?; + write_docker_boundary_file( + &directory.join(BOUNDARY_PRIVATE_KEY_FILE), + tls.sandbox_private_key_pem.as_bytes(), + ) + .await?; + write_docker_boundary_file( + &directory.join(BOUNDARY_CLIENT_CA_FILE), + tls.ca_certificate_pem.as_bytes(), + ) + .await?; + stage_docker_sandbox_bundle( + docker, + container_id, + config, + workload_identity, + &boundary_config, + DockerSandboxTls { + certificate: tls.sandbox_certificate_pem.as_bytes(), + private_key: tls.sandbox_private_key_pem.as_bytes(), + client_ca: tls.ca_certificate_pem.as_bytes(), + }, + &workspace_root, + ) + .await?; + let descriptor = provisioning + .topology + .descriptor(DRIVER_ADMITTED_BACKEND) + .map_err(|error| Status::internal(error.to_string()))?; + write_docker_boundary_file(&directory.join(TOPOLOGY_PAYLOAD_FILE), &descriptor.payload).await?; + let main_process_spec = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec( + sandbox.spec.as_ref(), + ) + .map_err(|error| Status::internal(format!("encode Docker main process spec: {error}")))?; + write_docker_boundary_file( + &directory.join(MAIN_PROCESS_SPEC_FILE), + main_process_spec.as_bytes(), + ) + .await?; + write_docker_boundary_file( + &directory.join(WORKSPACE_ROOT_FILE), + workspace_root.as_bytes(), + ) + .await?; + Ok(provisioning.topology) +} + +async fn docker_supervisor_bundle_archive( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result, Status> { + let directory = docker_boundary_state_dir(sandbox, config)?; + let topology = tokio::fs::read(directory.join(TOPOLOGY_PAYLOAD_FILE)) + .await + .map_err(|error| Status::internal(format!("read Docker topology payload: {error}")))?; + let token = tokio::fs::read(sandbox_token_host_path(sandbox, config)?) + .await + .map_err(|error| { + Status::failed_precondition(format!("read Docker sandbox JWT: {error}")) + })?; + if token.iter().all(u8::is_ascii_whitespace) { + return Err(Status::failed_precondition( + "Docker supervisor requires a sandbox JWT", + )); + } + let mut archive = tar::Builder::new(Vec::new()); + append_docker_archive_directory( + &mut archive, + ".openshell/channel/supervisor", + 0o700, + SUPERVISOR_UID, + SUPERVISOR_GID, + )?; + append_docker_archive_file( + &mut archive, + ".openshell/channel/supervisor/topology.payload", + 0o600, + SUPERVISOR_UID, + SUPERVISOR_GID, + &topology, + )?; + append_docker_archive_file( + &mut archive, + ".openshell/channel/supervisor/sandbox.jwt", + 0o600, + SUPERVISOR_UID, + SUPERVISOR_GID, + &token, + )?; + if let Some(tls) = &config.guest_tls { + append_docker_archive_directory( + &mut archive, + ".openshell/channel/supervisor/tls", + 0o700, + SUPERVISOR_UID, + SUPERVISOR_GID, + )?; + for (name, path) in [ + ("ca.pem", &tls.ca), + ("cert.pem", &tls.cert), + ("key.pem", &tls.key), + ] { + let contents = tokio::fs::read(path).await.map_err(|error| { + Status::internal(format!( + "read Docker supervisor TLS file {}: {error}", + path.display() + )) + })?; + append_docker_archive_file( + &mut archive, + &format!(".openshell/channel/supervisor/tls/{name}"), + 0o600, + SUPERVISOR_UID, + SUPERVISOR_GID, + &contents, + )?; + } + } + archive + .into_inner() + .map_err(|error| Status::internal(format!("finish Docker supervisor archive: {error}"))) +} + +async fn read_docker_boundary_topology( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> Result, Status> { + let path = docker_boundary_state_dir_by_id(sandbox_id, config)?.join(TOPOLOGY_PAYLOAD_FILE); + let bytes = match tokio::fs::read(&path).await { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(Status::internal(format!( + "read Docker boundary topology {}: {error}", + path.display() + ))); + } + }; + serde_json::from_slice(&bytes).map(Some).map_err(|error| { + Status::internal(format!( + "decode Docker boundary topology {}: {error}", + path.display() + )) + }) +} + +async fn stage_docker_supervisor_bundle( + docker: &Docker, + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, + archive: Vec, +) -> Result<(), Status> { + let stager_name = format!("{}-supervisor-stage", container_name_for_sandbox(sandbox)); + let _ = docker + .remove_container( + &stager_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + let created = docker + .create_container( + Some( + CreateContainerOptionsBuilder::default() + .name(stager_name.as_str()) + .build(), + ), + ContainerCreateBody { + image: Some(config.supervisor_image_id.clone()), + entrypoint: Some(vec![SUPERVISOR_IMAGE_CONTROL_BINARY_PATH.to_string()]), + labels: Some(docker_auxiliary_container_labels( + sandbox, + config, + LABEL_ISOLATION_ROLE_STAGING, + )), + host_config: Some(HostConfig { + network_mode: Some("none".to_string()), + mounts: Some(vec![Mount { + target: Some(BOUNDARY_MOUNT_PATH.to_string()), + source: Some(docker_channel_volume_name(sandbox, config)), + typ: Some(MountTypeEnum::VOLUME), + read_only: Some(false), + volume_options: Some(MountVolumeOptions { + no_copy: Some(true), + ..Default::default() + }), + ..Default::default() + }]), + cap_drop: Some(vec!["ALL".to_string()]), + security_opt: Some(vec!["no-new-privileges:true".to_string()]), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + .map_err(|error| { + Status::internal(format!( + "create Docker supervisor staging container: {error}" + )) + })?; + let options = UploadToContainerOptionsBuilder::default() + .path("/") + .copy_uidgid("true") + .build(); + let result = docker + .upload_to_container( + &created.id, + Some(options), + bollard::body_full(Bytes::from(archive)), + ) + .await + .map_err(|error| Status::internal(format!("stage Docker supervisor bundle: {error}"))); + let cleanup = docker + .remove_container( + &created.id, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await + .map_err(|error| { + Status::internal(format!( + "remove Docker supervisor staging container: {error}" + )) + }); + result?; + cleanup +} + +fn docker_auxiliary_container_labels( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, + role: &str, +) -> HashMap { + HashMap::from([ + ( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ), + (LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()), + (LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()), + ( + LABEL_SANDBOX_NAMESPACE.to_string(), + config.sandbox_namespace.clone(), + ), + (LABEL_ISOLATION_ROLE.to_string(), role.to_string()), + ]) +} + +async fn spawn_docker_control_process( + docker: &Docker, + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, + topology: &BoundaryTopology, + failure_context: DockerRuntimeFailureContext, +) -> Result { + let directory = docker_boundary_state_dir(sandbox, config)?; + let descriptor = topology + .descriptor(DRIVER_ADMITTED_BACKEND) + .map_err(|error| Status::internal(error.to_string()))?; + let main_process_spec = tokio::fs::read_to_string(directory.join(MAIN_PROCESS_SPEC_FILE)) + .await + .map_err(|error| Status::internal(format!("read Docker main process spec: {error}")))?; + let workspace_root = tokio::fs::read_to_string(directory.join(WORKSPACE_ROOT_FILE)) + .await + .map_err(|error| Status::internal(format!("read Docker workspace root: {error}")))?; + let supervisor_name = format!("{}-supervisor", container_name_for_sandbox(sandbox)); + let _ = docker + .remove_container( + &supervisor_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + let topology_path = format!("{SUPERVISOR_STATE_MOUNT_PATH}/topology.payload"); + let token_path = format!("{SUPERVISOR_STATE_MOUNT_PATH}/sandbox.jwt"); + let mut environment = vec![ + format!( + "{}={DRIVER_ADMITTED_BACKEND}", + openshell_core::sandbox_env::ADMITTED_ISOLATION_BACKEND + ), + format!( + "{}={main_process_spec}", + openshell_core::sandbox_env::MAIN_PROCESS_SPEC + ), + format!( + "{}={}", + openshell_core::sandbox_env::ENDPOINT, + config.supervisor_grpc_endpoint + ), + format!("{}={}", openshell_core::sandbox_env::SANDBOX_ID, sandbox.id), + format!("{}={}", openshell_core::sandbox_env::SANDBOX, sandbox.name), + format!( + "{}={token_path}", + openshell_core::sandbox_env::SANDBOX_TOKEN_FILE + ), + format!( + "{}=/run/openshell/ssh.sock", + openshell_core::sandbox_env::SSH_SOCKET_PATH + ), + format!( + "{}=/run/openshell/proxy-tls", + openshell_core::sandbox_env::PROXY_TLS_DIR + ), + format!( + "{}={}", + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY + ), + format!( + "{}={}", + openshell_core::sandbox_env::LOG_LEVEL, + openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level) + ), + format!( + "{}={}", + openshell_core::sandbox_env::TELEMETRY_ENABLED, + openshell_core::telemetry::enabled_env_value() + ), + ]; + if let Some(server_name) = config.gateway_tls_server_name.as_deref() { + environment.push(format!( + "{}={server_name}", + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME + )); + } + if config.guest_tls.is_some() { + environment.extend([ + format!( + "{}={SUPERVISOR_STATE_MOUNT_PATH}/tls/ca.pem", + openshell_core::sandbox_env::TLS_CA + ), + format!( + "{}={SUPERVISOR_STATE_MOUNT_PATH}/tls/cert.pem", + openshell_core::sandbox_env::TLS_CERT + ), + format!( + "{}={SUPERVISOR_STATE_MOUNT_PATH}/tls/key.pem", + openshell_core::sandbox_env::TLS_KEY + ), + ]); + } + let supervisor_archive = docker_supervisor_bundle_archive(sandbox, config).await?; + stage_docker_supervisor_bundle(docker, sandbox, config, supervisor_archive).await?; + let labels = HashMap::from([ + (LABEL_MANAGED_BY.to_string(), "openshell".to_string()), + (LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()), + (LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()), + ( + LABEL_SANDBOX_NAMESPACE.to_string(), + config.sandbox_namespace.clone(), + ), + ( + LABEL_ISOLATION_ROLE.to_string(), + LABEL_ISOLATION_ROLE_SUPERVISOR.to_string(), + ), + ]); + let create = ContainerCreateBody { + image: Some(config.supervisor_image_id.clone()), + user: Some(format!("{SUPERVISOR_UID}:{SUPERVISOR_GID}")), + entrypoint: Some(vec![SUPERVISOR_IMAGE_CONTROL_BINARY_PATH.to_string()]), + cmd: Some(vec![ + format!("--topology-backend-name={}", descriptor.backend_name), + "--topology-payload-file".to_string(), + topology_path.clone(), + "--workdir".to_string(), + workspace_root, + format!("--health-socket-path={SUPERVISOR_HEALTH_SOCKET_PATH}"), + ]), + env: Some(environment), + labels: Some(labels), + healthcheck: Some(HealthConfig { + test: Some(vec![ + "CMD".to_string(), + SUPERVISOR_IMAGE_CONTROL_BINARY_PATH.to_string(), + "health".to_string(), + "--socket".to_string(), + SUPERVISOR_HEALTH_SOCKET_PATH.to_string(), + ]), + interval: Some(SUPERVISOR_HEALTH_INTERVAL_NS), + timeout: Some(SUPERVISOR_HEALTH_TIMEOUT_NS), + retries: Some(3), + start_period: Some(SUPERVISOR_HEALTH_START_PERIOD_NS), + start_interval: Some(SUPERVISOR_HEALTH_INTERVAL_NS), + }), + host_config: Some(HostConfig { + network_mode: Some(config.network_name.clone()), + mounts: Some(vec![Mount { + target: Some(BOUNDARY_MOUNT_PATH.to_string()), + source: Some(docker_channel_volume_name(sandbox, config)), + typ: Some(MountTypeEnum::VOLUME), + read_only: Some(true), + volume_options: Some(MountVolumeOptions { + no_copy: Some(true), + ..Default::default() + }), + ..Default::default() + }]), + cap_drop: Some(vec!["ALL".to_string()]), + cap_add: None, + security_opt: Some(vec!["no-new-privileges:true".to_string()]), + readonly_rootfs: Some(true), + tmpfs: Some(HashMap::from([ + ( + "/run".to_string(), + format!( + "rw,noexec,nosuid,size=64m,uid={SUPERVISOR_UID},gid={SUPERVISOR_GID},mode=0700" + ), + ), + ( + "/tmp".to_string(), + "rw,noexec,nosuid,size=64m,mode=1777".to_string(), + ), + ( + "/var/log".to_string(), + format!( + "rw,noexec,nosuid,size=64m,uid={SUPERVISOR_UID},gid={SUPERVISOR_GID},mode=0700" + ), + ), + ])), + extra_hosts: Some(vec![ + format!( + "{HOST_OPENSHELL_INTERNAL}:{}", + docker_supervisor_host_alias(&config.gateway_route) + ), + format!( + "{HOST_DOCKER_INTERNAL}:{}", + docker_supervisor_host_alias(&config.gateway_route) + ), + ]), + restart_policy: None, + ..Default::default() + }), + ..Default::default() + }; + let created = docker + .create_container( + Some( + CreateContainerOptionsBuilder::default() + .name(supervisor_name.as_str()) + .build(), + ), + create, + ) + .await + .map_err(|error| { + Status::internal(format!("create Docker supervisor container: {error}")) + })?; + if let Err(error) = docker.start_container(&created.id, None).await { + let _ = docker + .remove_container( + &created.id, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + return Err(Status::internal(format!( + "start Docker supervisor container: {error}" + ))); + } + let sandbox_id = sandbox.id.clone(); + let (shutdown, mut shutdown_requested) = oneshot::channel(); + let intentional_shutdown = Arc::new(AtomicBool::new(false)); + let monitored_shutdown = intentional_shutdown.clone(); + let supervisor_id = created.id; + let monitored_supervisor_id = supervisor_id.clone(); + let monitored_docker = failure_context.docker.clone(); + let task = tokio::spawn(async move { + let wait = async { + let mut stream = monitored_docker.wait_container( + &monitored_supervisor_id, + None::, + ); + stream.next().await + }; + tokio::select! { + biased; + _ = &mut shutdown_requested => { + let _ = monitored_docker.stop_container( + &monitored_supervisor_id, + Some(StopContainerOptionsBuilder::default().t(5).build()), + ).await; + let _ = monitored_docker.remove_container( + &monitored_supervisor_id, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ).await; + } + result = wait => { + if monitored_shutdown.load(Ordering::Acquire) { + let _ = monitored_docker.remove_container( + &monitored_supervisor_id, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ).await; + return; + } + if matches!( + tokio::time::timeout( + SUPERVISOR_INTENTIONAL_SHUTDOWN_GRACE, + &mut shutdown_requested, + ) + .await, + Ok(Ok(())), + ) || monitored_shutdown.load(Ordering::Acquire) + { + let _ = monitored_docker.remove_container( + &monitored_supervisor_id, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ).await; + return; + } + let mut message = match result { + Some(Ok(status)) => { + warn!(%sandbox_id, status = status.status_code, "Docker supervisor container exited unexpectedly"); + format!("Docker supervisor container exited with status {}", status.status_code) + } + Some(Err(error)) => { + warn!(%sandbox_id, %error, "Failed to wait for Docker supervisor container"); + format!("failed to wait for Docker supervisor container: {error}") + } + None => "Docker supervisor wait stream ended unexpectedly".to_string(), + }; + let log_tail = + docker_container_log_tail(&monitored_docker, &monitored_supervisor_id).await; + if !log_tail.is_empty() { + write!(message, "; log tail: {log_tail}").ok(); + } + let sandbox_log_tail = + docker_container_log_tail(&monitored_docker, &failure_context.container_id) + .await; + if !sandbox_log_tail.is_empty() { + write!(message, "; sandbox log tail: {sandbox_log_tail}").ok(); + } + let _ = monitored_docker.remove_container( + &monitored_supervisor_id, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ).await; + // The gateway can close the supervisor session as soon as it + // commits Stopping. Re-check after collecting diagnostics so + // an overlapping driver stop cannot be published as an + // unexpected control failure. + if monitored_shutdown.load(Ordering::Acquire) { + return; + } + handle_docker_runtime_failure( + failure_context, + "ControlSupervisorExited", + message, + ) + .await; + }, + } + }); + let process = DockerControlProcess { + shutdown: Some(shutdown), + intentional_shutdown, + task, + }; + if let Err(error) = wait_for_docker_supervisor_ready(docker, &supervisor_id).await { + stop_docker_control_process(process).await; + return Err(error); + } + Ok(process) +} + +async fn wait_for_docker_supervisor_ready( + docker: &Docker, + supervisor_id: &str, +) -> Result<(), Status> { + let wait = async { + loop { + let inspected = docker + .inspect_container(supervisor_id, None) + .await + .map_err(|error| { + Status::internal(format!("inspect Docker supervisor container: {error}")) + })?; + let state = inspected.state.unwrap_or_default(); + match state.health.and_then(|health| health.status) { + Some(HealthStatusEnum::HEALTHY) => return Ok(()), + Some(HealthStatusEnum::UNHEALTHY) => { + let log_tail = docker_container_log_tail(docker, supervisor_id).await; + return Err(Status::unavailable(format!( + "Docker supervisor failed its readiness check{}", + format_log_tail(&log_tail) + ))); + } + _ if state.running == Some(false) => { + let log_tail = docker_container_log_tail(docker, supervisor_id).await; + return Err(Status::unavailable(format!( + "Docker supervisor exited before becoming ready{}", + format_log_tail(&log_tail) + ))); + } + _ => tokio::time::sleep(Duration::from_millis(100)).await, + } + } + }; + + if let Ok(result) = tokio::time::timeout(SUPERVISOR_READY_TIMEOUT, wait).await { + result + } else { + let log_tail = docker_container_log_tail(docker, supervisor_id).await; + Err(Status::deadline_exceeded(format!( + "Docker supervisor did not become ready within {} seconds{}", + SUPERVISOR_READY_TIMEOUT.as_secs(), + format_log_tail(&log_tail) + ))) + } +} + +fn format_log_tail(log_tail: &str) -> String { + if log_tail.is_empty() { + String::new() + } else { + format!("; log tail: {log_tail}") + } } -fn cleanup_sandbox_token_file_for_delete( - sandbox_id: &str, - pending: Option<&PendingSandboxRecord>, - config: &DockerDriverRuntimeConfig, +async fn docker_container_log_tail(docker: &Docker, container_id: &str) -> String { + const MAX_LOG_TAIL_BYTES: usize = 16 * 1024; + let options = LogsOptionsBuilder::default() + .stdout(true) + .stderr(true) + .tail("80") + .build(); + let mut stream = docker.logs(container_id, Some(options)); + let mut output = Vec::new(); + while let Some(result) = stream.next().await { + let Ok(chunk) = result else { + break; + }; + output.extend_from_slice(chunk.as_ref()); + if output.len() > MAX_LOG_TAIL_BYTES { + output.drain(..output.len() - MAX_LOG_TAIL_BYTES); + } + } + String::from_utf8_lossy(&output).trim().to_string() +} + +async fn handle_docker_runtime_failure( + context: DockerRuntimeFailureContext, + reason: &'static str, + message: String, ) { - if !sandbox_id.is_empty() { - cleanup_sandbox_token_file_by_id(sandbox_id, config); - } else if let Some(record) = pending { - cleanup_sandbox_token_file(&record.sandbox, config); + context.failures.lock().await.insert( + context.sandbox.id.clone(), + DockerRuntimeFailure { + reason, + message: message.clone(), + }, + ); + + let mut snapshot = pending_sandbox_snapshot( + &context.sandbox, + &context.sandbox_namespace, + error_condition(reason, &message), + false, + ); + if let Some(status) = snapshot.status.as_mut() { + status.instance_id.clone_from(&context.container_id); + } + let _ = context.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { + sandbox: Some(snapshot), + }, + )), + }); + let _ = context.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::PlatformEvent( + WatchSandboxesPlatformEvent { + sandbox_id: context.sandbox.id.clone(), + event: Some(platform_event( + "docker", + "Warning", + reason, + format!("{message}; stopping the isolated workload container"), + )), + }, + )), + }); + + match context + .docker + .stop_container( + &context.container_id, + Some( + StopContainerOptionsBuilder::default() + .t(docker_stop_timeout_secs(context.stop_timeout_secs)) + .build(), + ), + ) + .await + { + Ok(()) => info!( + sandbox_id = %context.sandbox.id, + container_id = %context.container_id, + "Stopped Docker sandbox after control supervisor failure" + ), + Err(error) if is_not_found_error(&error) || is_not_modified_error(&error) => {} + Err(error) => warn!( + sandbox_id = %context.sandbox.id, + container_id = %context.container_id, + %error, + "Failed to stop Docker sandbox after control supervisor failure" + ), + } +} + +async fn stop_docker_control_process(mut process: DockerControlProcess) { + process.intentional_shutdown.store(true, Ordering::Release); + if let Some(shutdown) = process.shutdown.take() { + let _ = shutdown.send(()); } + let _ = process.task.await; +} + +fn cleanup_docker_boundary_state(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) { + cleanup_docker_boundary_state_by_id(&sandbox.id, config); } -fn cleanup_sandbox_token_file_by_id(sandbox_id: &str, config: &DockerDriverRuntimeConfig) { - let Ok(path) = sandbox_token_host_path_by_id(sandbox_id, config) else { +fn cleanup_docker_boundary_state_by_id(sandbox_id: &str, config: &DockerDriverRuntimeConfig) { + let Ok(directory) = docker_boundary_state_dir_by_id(sandbox_id, config) else { return; }; - if let Err(err) = std::fs::remove_file(&path) - && err.kind() != std::io::ErrorKind::NotFound + if let Err(error) = std::fs::remove_dir_all(&directory) + && error.kind() != std::io::ErrorKind::NotFound { warn!( - sandbox_id = %sandbox_id, - path = %path.display(), - error = %err, - "Failed to remove Docker sandbox token file" + %sandbox_id, + path = %directory.display(), + %error, + "Failed to remove Docker boundary state directory" ); } - if let Some(dir) = path.parent() { - let _ = std::fs::remove_dir(dir); +} + +fn random_boundary_token() -> String { + let mut token = String::with_capacity(64); + for byte in rand::random::<[u8; 32]>() { + write!(&mut token, "{byte:02x}").expect("writing to String cannot fail"); } + token } -#[cfg(test)] -fn build_environment(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) -> Vec { - build_environment_for_oci_user(sandbox, config, "") +fn docker_child_environment(sandbox: &DriverSandbox) -> HashMap { + let mut environment = sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()) + .map_or_else(HashMap::new, |template| template.environment.clone()); + if let Some(spec) = sandbox.spec.as_ref() { + environment.extend(spec.environment.clone()); + } + for protected in [ + openshell_core::sandbox_env::ENDPOINT, + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME, + openshell_core::sandbox_env::MAIN_PROCESS_SPEC, + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + openshell_core::sandbox_env::OCI_IMAGE_USER, + openshell_core::sandbox_env::SANDBOX, + openshell_core::sandbox_env::SANDBOX_GID, + openshell_core::sandbox_env::SANDBOX_ID, + openshell_core::sandbox_env::SANDBOX_TOKEN, + openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, + openshell_core::sandbox_env::SANDBOX_UID, + openshell_core::sandbox_env::SSH_SOCKET_PATH, + openshell_core::sandbox_env::TLS_CA, + openshell_core::sandbox_env::TLS_CERT, + openshell_core::sandbox_env::TLS_KEY, + openshell_core::sandbox_env::USER_ENVIRONMENT, + ] { + environment.remove(protected); + } + environment } -fn build_environment_for_oci_user( +fn build_boundary_environment( sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, - oci_user: &str, ) -> Vec { - let mut environment = HashMap::from([ - ("HOME".to_string(), "/root".to_string()), - ("PATH".to_string(), SUPERVISOR_PATH.to_string()), - ("TERM".to_string(), "xterm".to_string()), - ( - "OPENSHELL_LOG_LEVEL".to_string(), - openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level), + vec![ + format!( + "{}={}", + openshell_core::sandbox_env::LOG_LEVEL, + openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level) ), - ]); - - if let Some(spec) = sandbox.spec.as_ref() { - let mut user_env = HashMap::new(); - if let Some(template) = spec.template.as_ref() { - user_env.extend(template.environment.clone()); - } - user_env.extend(spec.environment.clone()); - environment.extend(user_env.clone()); - if !user_env.is_empty() - && let Ok(json) = serde_json::to_string(&user_env) - { - environment.insert( - openshell_core::sandbox_env::USER_ENVIRONMENT.to_string(), - json, - ); - } - } - - environment.insert( - openshell_core::sandbox_env::ENDPOINT.to_string(), - config.grpc_endpoint.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_ID.to_string(), - sandbox.id.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX.to_string(), - sandbox.name.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SSH_SOCKET_PATH.to_string(), - config.ssh_socket_path.clone(), - ); - let main_process = - openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(sandbox.spec.as_ref()) - .expect("main process config serialization cannot fail"); - environment.insert( - openshell_core::sandbox_env::MAIN_PROCESS_SPEC.to_string(), - main_process, - ); - environment.insert( - openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), - openshell_core::telemetry::enabled_env_value().to_string(), - ); - environment.insert( - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY.to_string(), - ); - // The root supervisor executes namespace helpers during bootstrap; keep - // their search path driver-owned even when the template/spec set PATH. - environment.insert("PATH".to_string(), SUPERVISOR_PATH.to_string()); - if config.guest_tls.is_some() { - environment.insert( - openshell_core::sandbox_env::TLS_CA.to_string(), - TLS_CA_MOUNT_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_CERT.to_string(), - TLS_CERT_MOUNT_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_KEY.to_string(), - TLS_KEY_MOUNT_PATH.to_string(), - ); - } - if let Some(socket) = config.provider_spiffe_workload_api_socket.as_ref() - && let Ok(path) = - openshell_core::driver_utils::projected_provider_spiffe_socket_path(socket) - { - environment.insert( - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET.to_string(), - path, - ); - } - - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); - // Prevent user-supplied environment from overriding the TLS server name - // the supervisor verifies — a sandbox user who can redirect the gateway - // hostname could otherwise present a certificate for a name they control - // and intercept the sandbox JWT. - environment.remove(openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME); - environment.insert( - openshell_core::sandbox_env::OCI_IMAGE_USER.to_string(), - oci_user.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_UID.to_string(), - String::new(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_GID.to_string(), - String::new(), - ); - - // Gateway-minted sandbox JWT. Keep the raw bearer out of container - // metadata; the supervisor reads it from this driver-owned bind mount. - if let Some(spec) = sandbox.spec.as_ref() - && !spec.sandbox_token.is_empty() - { - environment.insert( - openshell_core::sandbox_env::SANDBOX_TOKEN_FILE.to_string(), - SANDBOX_TOKEN_MOUNT_PATH.to_string(), - ); - } - - let mut pairs = environment.into_iter().collect::>(); - pairs.sort_by(|left, right| left.0.cmp(&right.0)); - pairs - .into_iter() - .map(|(key, value)| format!("{key}={value}")) - .collect() + format!( + "{}={}", + openshell_core::sandbox_env::TELEMETRY_ENABLED, + openshell_core::telemetry::enabled_env_value() + ), + ] } fn docker_cdi_gpu_inventory(info: &SystemInfo) -> CdiGpuInventory { @@ -3129,6 +4965,14 @@ fn build_container_create_body_with_gpu_devices( .as_ref() .and_then(|spec| spec.template.as_ref()) .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; + let workload_identity = ResolvedWorkloadIdentity::new( + 1000, + 1000, + Vec::new(), + "test".to_string(), + template.image.clone(), + ) + .map_err(|error| Status::internal(error.to_string()))?; build_container_create_body_for_image( sandbox, config, @@ -3140,6 +4984,7 @@ fn build_container_create_body_with_gpu_devices( working_dir: String::new(), volumes: Vec::new(), }, + &workload_identity, ) } @@ -3149,6 +4994,7 @@ fn build_container_create_body_for_image( driver_config: &DockerSandboxDriverConfig, gpu_device_ids: Option<&[String]>, image: &DockerImageMetadata, + workload_identity: &ResolvedWorkloadIdentity, ) -> Result { let spec = sandbox .spec @@ -3161,7 +5007,7 @@ fn build_container_create_body_for_image( let resource_limits = docker_resource_limits(template)?; let workspace_root = driver_mounts::resolve_oci_workspace_root(&image.working_dir) .map_err(Status::failed_precondition)?; - driver_mounts::validate_workspace_control_path(&workspace_root, &config.ssh_socket_path) + driver_mounts::validate_workspace_control_path(&workspace_root, BOUNDARY_MOUNT_PATH) .map_err(Status::failed_precondition)?; for volume in &image.volumes { driver_mounts::validate_container_mount_target(volume).map_err(|error| { @@ -3174,7 +5020,7 @@ fn build_container_create_body_for_image( "image-declared volume '{volume}' masks OCI WorkingDir '{workspace_root}' before workspace validation" )) })?; - driver_mounts::validate_mount_control_path(volume, &config.ssh_socket_path) + driver_mounts::validate_mount_control_path(volume, BOUNDARY_MOUNT_PATH) .map_err(Status::failed_precondition)?; } for mount in &driver_config.mounts { @@ -3186,10 +5032,21 @@ fn build_container_create_body_for_image( }; driver_mounts::validate_workspace_mount_target(target, &workspace_root) .map_err(Status::failed_precondition)?; - driver_mounts::validate_mount_control_path(target, &config.ssh_socket_path) + driver_mounts::validate_mount_control_path(target, BOUNDARY_MOUNT_PATH) .map_err(Status::failed_precondition)?; } - let user_mounts = docker_driver_mounts(driver_config)?; + let mut user_mounts = docker_driver_mounts(driver_config)?; + user_mounts.push(Mount { + target: Some(BOUNDARY_MOUNT_PATH.to_string()), + source: Some(docker_channel_volume_name(sandbox, config)), + typ: Some(MountTypeEnum::VOLUME), + read_only: Some(false), + volume_options: Some(MountVolumeOptions { + no_copy: Some(true), + ..Default::default() + }), + ..Default::default() + }); let user_bind_strings = docker_driver_bind_strings(driver_config)?; let device_requests = gpu_device_ids.map(|device_ids| { vec![DeviceRequest { @@ -3209,30 +5066,39 @@ fn build_container_create_body_for_image( LABEL_SANDBOX_WORKSPACE.to_string(), sandbox.workspace.clone(), ); - // The list/get/find paths filter by `config.sandbox_label`, so use + // The list/get/find paths filter by `config.sandbox_namespace`, so use // the same value here. `DriverSandbox.namespace` is unset on the request // path (the gateway elides it), and using it would produce containers // that the driver itself cannot find afterwards. labels.insert( LABEL_SANDBOX_NAMESPACE.to_string(), - config.sandbox_label.clone(), + config.sandbox_namespace.clone(), + ); + labels.insert( + LABEL_ISOLATION_TOPOLOGY.to_string(), + LABEL_ISOLATION_TOPOLOGY_CAPABILITY_FREE.to_string(), + ); + labels.insert( + LABEL_ISOLATION_ROLE.to_string(), + LABEL_ISOLATION_ROLE_SANDBOX.to_string(), ); Ok(ContainerCreateBody { image: Some(image.id.clone()), - user: Some("0".to_string()), + user: Some(format!( + "{}:{}", + workload_identity.uid, workload_identity.gid + )), // The image workspace may need to be created or rejected by the // supervisor, so do not let the OCI runtime chdir there first. working_dir: Some("/".to_string()), - env: Some(build_environment_for_oci_user(sandbox, config, &image.user)), - entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]), - // Replace the image CMD with the supervisor's resolved workspace - // argument so Docker cannot append inherited image arguments. - cmd: { - let mut args = vec!["--workdir".to_string(), workspace_root]; - args.extend(docker_upstream_proxy_cli_args(&config.upstream_proxy)); - Some(args) - }, + env: Some(build_boundary_environment(sandbox, config)), + entrypoint: Some(vec![SANDBOX_BINARY_PATH.to_string()]), + // The image cannot append inherited arguments or select either role. + cmd: Some(vec![ + "--bootstrap".to_string(), + BOUNDARY_CONFIG_MOUNT_PATH.to_string(), + ]), labels: Some(labels), host_config: Some(HostConfig { nano_cpus: resource_limits.nano_cpus, @@ -3240,7 +5106,7 @@ fn build_container_create_body_for_image( pids_limit: docker_pids_limit(config.sandbox_pids_limit)?, device_requests, binds: { - let mut binds = build_binds(sandbox, config)?; + let mut binds = build_binds(sandbox, config); binds.extend(user_bind_strings); Some(binds) }, @@ -3248,29 +5114,33 @@ fn build_container_create_body_for_image( // Canonical main-process exit is terminal. Runtime restart would // silently create a new process generation behind the gateway. restart_policy: None, - cap_add: Some(vec![ - "SYS_ADMIN".to_string(), - "NET_ADMIN".to_string(), - "SYS_PTRACE".to_string(), - "SYSLOG".to_string(), - ]), - // The default is explicitly Unconfined because the supervisor - // needs mount operations commonly denied by docker-default. - security_opt: config - .app_armor_profile - .as_ref() - .and_then(AppArmorProfile::oci_security_opt) - .map(|option| vec![option]), - network_mode: Some(config.network_name.clone()), - extra_hosts: Some(docker_extra_hosts(&config.gateway_route)), - ..Default::default() - }), - networking_config: Some(NetworkingConfig { - endpoints_config: Some(HashMap::from([( - config.network_name.clone(), - EndpointSettings::default(), + group_add: Some( + workload_identity + .supplementary_gids + .iter() + .map(u32::to_string) + .collect(), + ), + cap_drop: Some(vec!["ALL".to_string()]), + cap_add: None, + security_opt: Some(vec!["no-new-privileges:true".to_string()]), + network_mode: Some("none".to_string()), + dns: Some(vec!["127.0.0.53".to_string()]), + tmpfs: Some(HashMap::from([( + "/run".to_string(), + format!( + "rw,noexec,nosuid,size=64m,uid={},gid={},mode=0755", + workload_identity.uid, workload_identity.gid + ), )])), + sysctls: Some(HashMap::from([( + "net.ipv4.ip_unprivileged_port_start".to_string(), + "0".to_string(), + )])), + extra_hosts: None, + ..Default::default() }), + networking_config: None, ..Default::default() }) } @@ -3289,16 +5159,35 @@ fn require_sandbox_identifier(sandbox_id: &str, sandbox_name: &str) -> Result<() Ok(()) } -fn docker_container_openshell_endpoint(endpoint: &str, host: &str, port: u16) -> String { - let Ok(mut url) = Url::parse(endpoint) else { - return endpoint.to_string(); +fn docker_host_openshell_endpoint( + endpoint: &str, + route: &DockerGatewayRoute, +) -> CoreResult { + let mut url = Url::parse(endpoint) + .map_err(|error| Error::config(format!("invalid docker grpc_endpoint: {error}")))?; + if !matches!( + url.host_str(), + Some(HOST_OPENSHELL_INTERNAL | HOST_DOCKER_INTERNAL) + ) { + return Ok(url.to_string()); + } + let host = match route { + DockerGatewayRoute::Bridge { bind_address, .. } => bind_address.ip(), + DockerGatewayRoute::HostGateway => IpAddr::V4(Ipv4Addr::LOCALHOST), }; + url.set_host(Some(&host.to_string())).map_err(|error| { + Error::config(format!( + "failed to map Docker gateway alias to its host listener: {error}" + )) + })?; + Ok(url.to_string()) +} - if url.set_host(Some(host)).is_ok() && url.set_port(Some(port)).is_ok() { - return url.to_string(); +fn docker_supervisor_host_alias(route: &DockerGatewayRoute) -> String { + match route { + DockerGatewayRoute::Bridge { bind_address } => bind_address.ip().to_string(), + DockerGatewayRoute::HostGateway => "host-gateway".to_string(), } - - endpoint.to_string() } fn docker_network_name(config: &DockerComputeConfig) -> String { @@ -3346,7 +5235,6 @@ fn docker_gateway_route_for_host( if let Some(host_alias_ip) = host_gateway_ip { return DockerGatewayRoute::Bridge { bind_address: SocketAddr::new(host_alias_ip, port), - host_alias_ip, }; } @@ -3355,7 +5243,6 @@ fn docker_gateway_route_for_host( } else { DockerGatewayRoute::Bridge { bind_address: SocketAddr::new(bridge_gateway_ip, port), - host_alias_ip: bridge_gateway_ip, } } } @@ -3420,19 +5307,6 @@ fn uses_host_gateway_alias(info: &SystemInfo) -> bool { }) } -fn docker_extra_hosts(route: &DockerGatewayRoute) -> Vec { - match route { - DockerGatewayRoute::Bridge { host_alias_ip, .. } => vec![ - format!("{HOST_DOCKER_INTERNAL}:{host_alias_ip}"), - format!("{HOST_OPENSHELL_INTERNAL}:{host_alias_ip}"), - ], - DockerGatewayRoute::HostGateway => vec![ - format!("{HOST_DOCKER_INTERNAL}:host-gateway"), - format!("{HOST_OPENSHELL_INTERNAL}:host-gateway"), - ], - } -} - async fn ensure_bridge_network(docker: &Docker, network_name: &str) -> CoreResult { match docker.inspect_network(network_name, None).await { Ok(network) => return validate_bridge_network(network_name, &network), @@ -3545,55 +5419,26 @@ fn docker_resource_limits( }) } -fn validate_sandbox_pids_limit(value: Option) -> CoreResult<()> { - if value.is_some_and(|limit| limit.get() < 0) { - return Err(Error::config( - "docker sandbox_pids_limit must be positive when set", - )); - } - Ok(()) -} - -fn validate_image_pull_policy(policy: ImagePullPolicy) -> CoreResult<()> { - if policy == ImagePullPolicy::Newer { - return Err(Error::config( - "docker image_pull_policy = \"newer\" is supported only by the Podman compute driver", - )); - } - Ok(()) -} - -fn validate_docker_app_armor_profile( - profile: Option<&AppArmorProfile>, - info: &SystemInfo, -) -> CoreResult<()> { - let requires_apparmor = matches!( - profile, - Some(AppArmorProfile::RuntimeDefault | AppArmorProfile::Localhost(_)) - ); - if !requires_apparmor { - return Ok(()); - } - let available = info.security_options.as_ref().is_some_and(|options| { - options - .iter() - .any(|option| option.to_ascii_lowercase().contains("apparmor")) - }); - if !available { +fn validate_sandbox_pids_limit(value: i64) -> CoreResult<()> { + if value < 0 { return Err(Error::config( - "app_armor_profile requires AppArmor, but Docker reports it is unavailable; enable AppArmor on the daemon host or set app_armor_profile = \"Unconfined\" explicitly", + "docker sandbox_pids_limit must be zero or greater", )); } Ok(()) } -fn docker_pids_limit(value: Option) -> Result, Status> { - if value.is_some_and(|limit| limit.get() < 0) { +fn docker_pids_limit(value: i64) -> Result, Status> { + if value < 0 { return Err(Status::failed_precondition( - "docker sandbox_pids_limit must be positive when set", + "docker sandbox_pids_limit must be zero or greater", )); } - Ok(value.map(std::num::NonZeroI64::get)) + if value == 0 { + Ok(None) + } else { + Ok(Some(value)) + } } #[allow(clippy::cast_possible_truncation)] @@ -3725,10 +5570,8 @@ fn driver_status_from_summary( /// Refine an exited Docker sandbox's `Ready` condition from inspected state. /// -/// A workspace-validation exit is reported distinctly so users can repair the -/// OCI working directory rather than diagnose a generic crash. A signal kill -/// (exit 137/143 = SIGKILL/SIGTERM, not OOM) is the signature of a -/// machine/daemon restart terminating a running container. Reclassify it from +/// A signal kill (exit 137/143 = SIGKILL/SIGTERM, not OOM) is the signature of +/// a machine/daemon restart terminating a running container. Reclassify it from /// the generic terminal `ContainerExited` to the recoverable /// `ContainerRuntimeRestart` so gateway startup can revive it. OOM kills and /// ordinary application exits stay `ContainerExited` and terminal. @@ -3736,7 +5579,7 @@ fn apply_docker_exit_classification(sandbox: &mut DriverSandbox, state: &Contain if state.oom_killed == Some(true) { return; } - let Some(code) = state.exit_code else { + let Some(code) = state.exit_code.filter(|&code| matches!(code, 137 | 143)) else { return; }; let Some(condition) = sandbox @@ -3749,13 +5592,8 @@ fn apply_docker_exit_classification(sandbox: &mut DriverSandbox, state: &Contain if condition.reason != CONDITION_EXITED { return; } - if code == i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED) { - condition.reason = CONDITION_WORKSPACE_VALIDATION_FAILED.to_string(); - condition.message = "OCI WorkingDir is not usable by the sandbox identity".to_string(); - } else if matches!(code, 137 | 143) { - condition.reason = CONDITION_RUNTIME_RESTART.to_string(); - condition.message = format!("Container terminated by signal (exit code {code})"); - } + condition.reason = CONDITION_RUNTIME_RESTART.to_string(); + condition.message = format!("Container terminated by signal (exit code {code})"); } fn container_ready_condition( @@ -3854,12 +5692,23 @@ fn label_filters(values: impl IntoIterator) -> HashMap, +) -> HashMap> { + let mut values = vec![format!( + "{LABEL_ISOLATION_ROLE}={LABEL_ISOLATION_ROLE_SANDBOX}" + )]; + values.extend(extra_values); + managed_resource_label_filters(sandbox_namespace, values) +} + +fn managed_resource_label_filters( + sandbox_namespace: &str, extra_values: impl IntoIterator, ) -> HashMap> { let mut values = vec![ format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}"), - format!("{LABEL_SANDBOX_NAMESPACE}={sandbox_label}"), + format!("{LABEL_SANDBOX_NAMESPACE}={sandbox_namespace}"), ]; values.extend(extra_values); label_filters(values) @@ -3934,189 +5783,6 @@ fn sanitize_docker_name(value: &str) -> String { .to_string() } -fn normalize_docker_arch(arch: &str) -> String { - match arch { - "x86_64" => "amd64".to_string(), - "aarch64" => "arm64".to_string(), - other => other.to_ascii_lowercase(), - } -} - -#[derive(Debug, Eq, PartialEq)] -enum SupervisorBinSource { - Binary(PathBuf), - Image(String), -} - -fn resolve_supervisor_bin_source( - docker_config: &DockerComputeConfig, - current_exe: Option<&Path>, - target_candidates: &[PathBuf], -) -> CoreResult { - // Tier 1: explicit supervisor_bin in [openshell.drivers.docker]. - if let Some(path) = docker_config.supervisor_bin.clone() { - let path = canonicalize_existing_file(&path, "docker supervisor binary")?; - validate_linux_elf_binary(&path).map_err(Error::config)?; - return Ok(SupervisorBinSource::Binary(path)); - } - - // Tier 2: explicit supervisor_image in [openshell.drivers.docker]. - // A configured image should be the source of truth even when a local - // developer build is present under target/. - if let Some(image) = docker_config.supervisor_image.clone() { - return Ok(SupervisorBinSource::Image(image)); - } - - // Tier 3: sibling `openshell-sandbox` next to the running gateway - // (release artifact layout). Linux-only because the sibling must be a - // Linux ELF to bind-mount into a Linux container. - if cfg!(target_os = "linux") - && let Some(current_exe) = current_exe - && let Some(parent) = current_exe.parent() - { - let sibling = parent.join("openshell-sandbox"); - if sibling.is_file() { - let path = canonicalize_existing_file(&sibling, "docker supervisor binary")?; - if validate_linux_elf_binary(&path).is_ok() { - return Ok(SupervisorBinSource::Binary(path)); - } - } - } - - // Tier 4: local cargo target build (developer workflow). Preferred - // over the default registry image when available because it matches - // whatever the developer just built. - for candidate in target_candidates { - if candidate.is_file() { - let path = canonicalize_existing_file(candidate, "docker supervisor binary")?; - if validate_linux_elf_binary(&path).is_ok() { - return Ok(SupervisorBinSource::Binary(path)); - } - } - } - - // Tier 5: pull the release-matched default supervisor image and extract - // the binary to a host-side cache keyed by image content digest. - Ok(SupervisorBinSource::Image( - openshell_core::config::default_supervisor_image(), - )) -} - -pub(crate) async fn resolve_supervisor_bin( - docker: &Docker, - docker_config: &DockerComputeConfig, - daemon_arch: &str, -) -> CoreResult { - let current_exe = - if cfg!(target_os = "linux") - && docker_config.supervisor_bin.is_none() - && docker_config.supervisor_image.is_none() - { - Some(std::env::current_exe().map_err(|err| { - Error::config(format!("failed to resolve current executable: {err}")) - })?) - } else { - None - }; - let target_candidates = linux_supervisor_candidates(daemon_arch); - - match resolve_supervisor_bin_source(docker_config, current_exe.as_deref(), &target_candidates)? - { - SupervisorBinSource::Binary(path) => Ok(path), - SupervisorBinSource::Image(image) => { - extract_supervisor_bin_from_image(docker, &image).await - } - } -} - -fn linux_supervisor_candidates(daemon_arch: &str) -> Vec { - match daemon_arch { - "arm64" => vec![PathBuf::from( - "target/aarch64-unknown-linux-gnu/release/openshell-sandbox", - )], - "amd64" => vec![PathBuf::from( - "target/x86_64-unknown-linux-gnu/release/openshell-sandbox", - )], - _ => Vec::new(), - } -} - -/// Pull the supervisor image (if not already local), extract -/// `/openshell-sandbox` to a host cache keyed by the image's content -/// digest, and return the cache path. -/// -/// The extraction is atomic: the binary is written to a sibling temp file -/// inside the digest-keyed directory and renamed into place, so concurrent -/// gateway starts don't observe a partial file. -async fn extract_supervisor_bin_from_image(docker: &Docker, image: &str) -> CoreResult { - let refresh_attempted = if supervisor_image_should_refresh(image) { - info!(image = image, "Refreshing mutable docker supervisor image"); - match pull_supervisor_image(docker, image).await { - Ok(()) => true, - Err(err) => { - warn!( - image = image, - error = %err, - "failed to refresh mutable docker supervisor image; falling back to local image if present", - ); - true - } - } - } else { - false - }; - - // Inspect first to see if the image is already present; only pull on miss. - let inspect = match docker.inspect_image(image).await { - Ok(inspect) => inspect, - Err(err) if is_not_found_error(&err) && !refresh_attempted => { - info!(image = image, "Pulling docker supervisor image"); - pull_supervisor_image(docker, image).await?; - docker.inspect_image(image).await.map_err(|err| { - Error::config(format!( - "failed to inspect docker supervisor image '{image}' after pull: {err}", - )) - })? - } - Err(err) if is_not_found_error(&err) => { - return Err(Error::config(format!( - "docker supervisor image '{image}' is not present locally after refresh attempt", - ))); - } - Err(err) => { - return Err(Error::config(format!( - "failed to inspect docker supervisor image '{image}': {err}", - ))); - } - }; - - let digest = inspect.id.clone().ok_or_else(|| { - Error::config(format!( - "docker supervisor image '{image}' inspect response has no Id", - )) - })?; - - let cache_path = - openshell_core::driver_utils::supervisor_cache_path("docker-supervisor", &digest) - .map_err(Error::config)?; - if cache_path.is_file() { - validate_linux_elf_binary(&cache_path).map_err(Error::config)?; - return Ok(cache_path); - } - - info!( - image = image, - digest = digest, - cache_path = %cache_path.display(), - "Extracting supervisor binary from image to host cache", - ); - - let binary_bytes = extract_supervisor_binary_bytes(docker, image).await?; - write_cache_binary_atomic(&cache_path, &binary_bytes).map_err(Error::config)?; - validate_linux_elf_binary(&cache_path).map_err(Error::config)?; - Ok(cache_path) -} - async fn pull_supervisor_image(docker: &Docker, image: &str) -> CoreResult<()> { let mut stream = docker.create_image( Some(CreateImageOptions { @@ -4136,10 +5802,55 @@ async fn pull_supervisor_image(docker: &Docker, image: &str) -> CoreResult<()> { Ok(()) } +async fn ensure_supervisor_container_image(docker: &Docker, image: &str) -> CoreResult { + let local_image_present = docker.inspect_image(image).await.is_ok(); + if supervisor_image_should_refresh(image) { + info!(image = image, "Refreshing mutable docker supervisor image"); + if let Err(error) = pull_supervisor_image(docker, image).await { + if !local_image_present { + return Err(error); + } + warn!( + image = image, + error = %error, + "failed to refresh mutable Docker supervisor image; using the local image", + ); + } + } else if !local_image_present { + pull_supervisor_image(docker, image).await?; + } + let inspect = docker.inspect_image(image).await.map_err(|error| { + Error::config(format!( + "failed to inspect Docker supervisor image '{image}': {error}" + )) + })?; + inspect.id.filter(|id| !id.is_empty()).ok_or_else(|| { + Error::config(format!( + "Docker supervisor image '{image}' has no immutable image ID" + )) + }) +} + /// Create a short-lived container from `image`, stream out the supervisor /// binary as a tar archive, and return the untarred file bytes. The /// container is always removed, even on error paths. async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreResult> { + let bytes = + extract_supervisor_path_archive(docker, image, SUPERVISOR_IMAGE_BINARY_PATH, true).await?; + if !bytes.starts_with(b"\x7fELF") { + return Err(Error::config(format!( + "Docker supervisor image '{image}' contains an invalid sandbox binary" + ))); + } + Ok(bytes) +} + +async fn extract_supervisor_path_archive( + docker: &Docker, + image: &str, + path: &str, + extract_single_file: bool, +) -> CoreResult> { let container_name = temp_extract_container_name(); docker .create_container( @@ -4163,7 +5874,8 @@ async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreRe })?; // Always tear down the extractor container, even if extraction fails. - let result = download_binary_from_container(docker, &container_name).await; + let result = + download_path_from_container(docker, &container_name, path, extract_single_file).await; if let Err(remove_err) = docker .remove_container( &container_name, @@ -4180,12 +5892,14 @@ async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreRe result } -async fn download_binary_from_container( +async fn download_path_from_container( docker: &Docker, container_name: &str, + path: &str, + extract_single_file: bool, ) -> CoreResult> { let options = DownloadFromContainerOptionsBuilder::default() - .path(SUPERVISOR_IMAGE_BINARY_PATH) + .path(path) .build(); let mut stream = docker.download_from_container(container_name, Some(options)); @@ -4199,11 +5913,15 @@ async fn download_binary_from_container( tar_bytes.extend_from_slice(&chunk); } - extract_first_tar_entry(&tar_bytes).map_err(|err| { - Error::config(format!( - "failed to extract supervisor binary from tar archive returned by '{container_name}': {err}", - )) - }) + if extract_single_file { + extract_first_tar_entry(&tar_bytes).map_err(|err| { + Error::config(format!( + "failed to extract supervisor binary from tar archive returned by '{container_name}': {err}", + )) + }) + } else { + Ok(tar_bytes) + } } fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult { @@ -4223,8 +5941,8 @@ fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult bool { docker_config.guest_tls_ca.is_some() - || docker_config.guest_tls_cert.is_some() - || docker_config.guest_tls_key.is_some() + && docker_config.guest_tls_cert.is_some() + && docker_config.guest_tls_key.is_some() } pub(crate) fn docker_guest_tls_paths( @@ -4298,6 +6016,16 @@ fn is_conflict_error(err: &BollardError) -> bool { ) } +fn is_removal_in_progress_error(err: &BollardError) -> bool { + matches!( + err, + BollardError::DockerResponseServerError { + status_code: 409, + message, + } if message.contains("removal of container") && message.contains("is already in progress") + ) +} + fn is_not_modified_error(err: &BollardError) -> bool { matches!( err, diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 89dd7c5216..ba86bf888a 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -4,9 +4,8 @@ use super::*; use openshell_core::config::DEFAULT_SERVER_PORT; use openshell_core::driver_utils::{ - CONDITION_WORKSPACE_VALIDATION_FAILED, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, - LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, - SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED, supervisor_cache_path_with_base, + LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, + LABEL_SANDBOX_NAMESPACE, }; use openshell_core::progress::{ PROGRESS_ACTIVE_DETAIL_KEY, PROGRESS_ACTIVE_STEP_KEY, PROGRESS_COMPLETE_LABEL_KEY, @@ -16,16 +15,13 @@ use openshell_core::progress::{ use openshell_core::proto::compute::v1::{ DriverResourceRequirements, DriverSandboxSpec, DriverSandboxTemplate, GetGatewayListenerRequirementsRequest, GpuResourceRequirements, ResourceRequirements, - gateway_listener_requirement::Selector, + WorkloadIdentityRequest, gateway_listener_requirement::Selector, }; use std::fs; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use std::sync::{Arc, LazyLock, Mutex}; +use std::sync::Arc; use tempfile::TempDir; -const TLS_MOUNT_DIR: &str = "/etc/openshell/tls/client"; -static ENV_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); - fn test_sandbox() -> DriverSandbox { // Mirrors the gateway-supplied request: the public `Sandbox` API no // longer carries `namespace`, so the gateway elides the field and the @@ -98,25 +94,25 @@ fn gpu_resources(count: Option) -> ResourceRequirements { fn runtime_config() -> DockerDriverRuntimeConfig { DockerDriverRuntimeConfig { default_image: "image:latest".to_string(), - image_pull_policy: ImagePullPolicy::IfNotPresent, - sandbox_label: "default".to_string(), - grpc_endpoint: "https://localhost:8443".to_string(), - network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), + image_pull_policy: String::new(), + sandbox_namespace: "default".to_string(), gateway_route: DockerGatewayRoute::Bridge { bind_address: SocketAddr::new( IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), DEFAULT_SERVER_PORT, ), - host_alias_ip: IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), }, gateway_callback_bind_address: Some(SocketAddr::new( IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), DEFAULT_SERVER_PORT, )), - ssh_socket_path: "/run/openshell/ssh.sock".to_string(), stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, log_level: "info".to_string(), - supervisor_bin: PathBuf::from("/tmp/openshell-sandbox"), + sandbox_binary: Arc::new(b"\x7fELFtest".to_vec()), + supervisor_image_id: "sha256:supervisor-test".to_string(), + network_name: "openshell-test".to_string(), + supervisor_grpc_endpoint: "https://host.openshell.internal:8443".to_string(), + gateway_tls_server_name: None, guest_tls: Some(DockerGuestTlsPaths { ca: PathBuf::from("/tmp/ca.crt"), cert: PathBuf::from("/tmp/tls.crt"), @@ -127,159 +123,20 @@ fn runtime_config() -> DockerDriverRuntimeConfig { cdi_supported: false, wsl_all_gpu_fallback_enabled: false, }, - sandbox_pids_limit: None, + sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, enable_bind_mounts: false, - upstream_proxy: UpstreamProxyConfig::default(), - provider_spiffe_workload_api_socket: None, - app_armor_profile: Some(AppArmorProfile::Unconfined), - } -} - -#[test] -fn docker_config_uses_canonical_sandbox_label_name() { - let config: DockerComputeConfig = - serde_json::from_value(serde_json::json!({ "sandbox_label": "tenant-a" })).unwrap(); - assert_eq!(config.sandbox_label, "tenant-a"); - - let serialized = serde_json::to_value(config).unwrap(); - assert_eq!(serialized["sandbox_label"], "tenant-a"); - assert!(serialized.get("sandbox_namespace").is_none()); -} - -#[test] -fn docker_config_rejects_legacy_sandbox_namespace() { - let error = serde_json::from_value::(serde_json::json!({ - "sandbox_namespace": "tenant-a" - })) - .expect_err("legacy sandbox_namespace must be rejected"); - assert!(error.to_string().contains("sandbox_namespace")); -} - -#[test] -fn docker_config_keeps_explicit_unconfined_apparmor_default() { - let config: DockerComputeConfig = serde_json::from_value(serde_json::json!({})) - .expect("default Docker config should deserialize"); - assert_eq!(config.app_armor_profile, Some(AppArmorProfile::Unconfined)); - let serialized = serde_json::to_value(config).expect("config should serialize"); - assert_eq!(serialized["app_armor_profile"], "Unconfined"); -} - -#[test] -fn docker_config_defaults_to_driver_owned_pids_limit() { - let config: DockerComputeConfig = serde_json::from_value(serde_json::json!({})) - .expect("default Docker config should deserialize"); - assert_eq!( - config.sandbox_pids_limit.map(std::num::NonZeroI64::get), - Some(openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT) - ); -} - -#[test] -fn docker_config_rejects_invalid_pids_limits() { - let zero = serde_json::from_value::(serde_json::json!({ - "sandbox_pids_limit": 0 - })) - .expect_err("zero PID limit must be rejected"); - assert!(zero.to_string().contains("invalid value: integer `0`")); - - let negative: DockerComputeConfig = serde_json::from_value(serde_json::json!({ - "sandbox_pids_limit": -1 - })) - .expect("nonzero integer deserializes before semantic validation"); - let error = validate_sandbox_pids_limit(negative.sandbox_pids_limit).unwrap_err(); - assert!(error.to_string().contains("must be positive")); -} - -#[test] -fn docker_rejects_newer_image_pull_policy() { - let error = validate_image_pull_policy(ImagePullPolicy::Newer).unwrap_err(); - assert!(error.to_string().contains("supported only by the Podman")); -} - -#[test] -fn docker_apparmor_profiles_render_and_require_daemon_capability() { - for (profile, expected) in [ - (AppArmorProfile::RuntimeDefault, None), - ( - AppArmorProfile::Unconfined, - Some(vec!["apparmor=unconfined".to_string()]), - ), - ( - AppArmorProfile::Localhost("openshell-supervisor".to_string()), - Some(vec!["apparmor=openshell-supervisor".to_string()]), - ), - ] { - let mut config = runtime_config(); - config.app_armor_profile = Some(profile.clone()); - let body = build_container_create_body(&test_sandbox(), &config).unwrap(); - assert_eq!(body.host_config.unwrap().security_opt, expected); - } - - let unavailable = SystemInfo::default(); - assert!( - validate_docker_app_armor_profile(Some(&AppArmorProfile::Unconfined), &unavailable).is_ok() - ); - for confined in [ - AppArmorProfile::RuntimeDefault, - AppArmorProfile::Localhost("openshell-supervisor".to_string()), - ] { - let error = validate_docker_app_armor_profile(Some(&confined), &unavailable) - .expect_err("confined profile requires daemon AppArmor support"); - assert!( - error - .to_string() - .contains("Docker reports it is unavailable") - ); } - - let available = SystemInfo { - security_options: Some(vec!["name=apparmor".to_string()]), - ..Default::default() - }; - assert!( - validate_docker_app_armor_profile( - Some(&AppArmorProfile::Localhost( - "openshell-supervisor".to_string() - )), - &available - ) - .is_ok() - ); } -#[test] -fn docker_config_uses_shared_proxy_contract_and_explicit_apparmor_default() { - let config: DockerComputeConfig = toml::from_str( - r#" -https_proxy = "http://proxy.example:8080" -no_proxy = ".svc" -proxy_auth_file = "/run/secrets/proxy-auth" -proxy_auth_allow_insecure = true -app_armor_profile = "Localhost/openshell-supervisor" -provider_spiffe_workload_api_socket = "/run/spire/agent.sock" -"#, +fn test_workload_identity() -> ResolvedWorkloadIdentity { + ResolvedWorkloadIdentity::new( + 1234, + 1235, + vec![1236], + "test".to_string(), + "sha256:immutable".to_string(), ) - .unwrap(); - assert_eq!( - config.upstream_proxy.https_proxy.as_deref(), - Some("http://proxy.example:8080") - ); - assert_eq!( - config.app_armor_profile, - Some(AppArmorProfile::Localhost( - "openshell-supervisor".to_string() - )) - ); - assert!(config.upstream_proxy.validate().is_ok()); - assert!( - openshell_core::driver_utils::validate_provider_spiffe_unix_socket( - config - .provider_spiffe_workload_api_socket - .as_deref() - .unwrap() - ) - .is_ok() - ); + .unwrap() } fn json_struct(value: serde_json::Value) -> prost_types::Struct { @@ -314,12 +171,14 @@ fn test_driver_with_config(config: DockerDriverRuntimeConfig) -> DockerComputeDr ), config, events: broadcast::channel(WATCH_BUFFER).0, - pending: Arc::new(tokio::sync::Mutex::new(HashMap::new())), + pending: Arc::new(Mutex::new(HashMap::new())), gpu_selector: Arc::new(CdiGpuDefaultSelector::new( CdiGpuInventory::default(), wsl_all_gpu_fallback_enabled, )), lifecycle_event_fences: DockerLifecycleEventFences::default(), + control_processes: Arc::new(Mutex::new(HashMap::new())), + runtime_failures: Arc::new(Mutex::new(HashMap::new())), } } @@ -361,39 +220,16 @@ fn request_with_traceparent(message: T) -> Request { request } -async fn fake_docker_with_no_containers() -> (String, JoinHandle<()>) { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { - while let Ok((mut stream, _)) = listener.accept().await { - openshell_core::net::set_tcp_nodelay_best_effort(&stream); - let mut scratch = [0_u8; 4096_usize]; - let _ = stream.read(&mut scratch).await; - let _ = stream - .write_all( - b"HTTP/1.1 200 OK\r\n\ - Content-Type: application/json\r\n\ - Content-Length: 2\r\n\r\n[]", - ) - .await; - let _ = stream.flush().await; - } - }); - (format!("http://{address}"), server) -} - async fn standalone_traced_client() -> ( TestDriverClient, - tokio::sync::oneshot::Sender<()>, + oneshot::Sender<()>, JoinHandle>, ) { use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); - let (shutdown, shutdown_rx) = tokio::sync::oneshot::channel(); + let (shutdown, shutdown_rx) = oneshot::channel(); let service = ComputeDriverService::new(test_driver_with_config(runtime_config())); let server = tokio::spawn(async move { tonic::transport::Server::builder() @@ -480,6 +316,78 @@ async fn tracing_standalone_rpc_layer_propagates_context_and_records_errors() { provider.shutdown().unwrap(); } +#[tokio::test] +async fn control_failure_overrides_running_container_readiness() { + let driver = test_driver_with_config(runtime_config()); + driver.runtime_failures.lock().await.insert( + "sbx-123".to_string(), + DockerRuntimeFailure { + reason: "ControlSupervisorExited", + message: "control exited unexpectedly".to_string(), + }, + ); + let mut sandbox = pending_sandbox_snapshot( + &test_sandbox(), + "default", + DriverCondition { + r#type: "Ready".to_string(), + status: "True".to_string(), + reason: "BackendReady".to_string(), + message: "Container is running".to_string(), + last_transition_time: String::new(), + }, + false, + ); + + driver.apply_runtime_failure(&mut sandbox).await; + + let ready = sandbox + .status + .unwrap() + .conditions + .into_iter() + .find(|condition| condition.r#type == "Ready") + .expect("ready condition"); + assert_eq!(ready.status, "False"); + assert_eq!(ready.reason, "ControlSupervisorExited"); + assert!(ready.message.contains("control exited unexpectedly")); +} + +#[tokio::test] +async fn control_failure_does_not_hide_a_terminal_container_exit() { + let driver = test_driver_with_config(runtime_config()); + driver.runtime_failures.lock().await.insert( + "sbx-123".to_string(), + DockerRuntimeFailure { + reason: "ControlSupervisorExited", + message: "control exited unexpectedly".to_string(), + }, + ); + let mut sandbox = pending_sandbox_snapshot( + &test_sandbox(), + "default", + DriverCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: CONDITION_EXITED.to_string(), + message: "Container exited".to_string(), + last_transition_time: String::new(), + }, + false, + ); + + driver.apply_runtime_failure(&mut sandbox).await; + + let ready = sandbox + .status + .unwrap() + .conditions + .into_iter() + .find(|condition| condition.r#type == "Ready") + .expect("ready condition"); + assert_eq!(ready.reason, CONDITION_EXITED); +} + #[tokio::test] async fn tracing_in_process_service_preserves_the_driver_rpc_server_boundary() { use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; @@ -698,8 +606,7 @@ async fn tracing_direct_start_exports_a_docker_start_span() { let subscriber = tracing_subscriber::registry().with(otel_tracing::TRACING.layer(&provider)); let driver = test_driver_with_config(runtime_config()); - DockerComputeDriver::start_sandbox(&driver, "", "") - .with_subscriber(subscriber) + Box::pin(DockerComputeDriver::start_sandbox(&driver, "", "").with_subscriber(subscriber)) .await .expect_err("missing identifier should fail"); provider.force_flush().unwrap(); @@ -729,17 +636,19 @@ async fn tracing_image_preparation_failure_exports_nested_failed_spans() { .build(); let subscriber = tracing_subscriber::registry().with(otel_tracing::TRACING.layer(&provider)); let mut config = runtime_config(); - config.image_pull_policy = ImagePullPolicy::Newer; + config.image_pull_policy = "unsupported".to_string(); let driver = test_driver_with_config(config); async { - driver - .provision_sandbox_inner(&test_sandbox()) - .instrument(tracing::info_span!( - "docker.provision", - otel.status_code = tracing::field::Empty - )) - .await + Box::pin( + driver + .provision_sandbox_inner(&test_sandbox()) + .instrument(tracing::info_span!( + "docker.provision", + otel.status_code = tracing::field::Empty + )), + ) + .await } .with_subscriber(subscriber) .await @@ -1017,34 +926,6 @@ async fn host_gateway_route_reports_ipv4_loopback_callback_listener() { ); } -#[test] -fn container_visible_endpoint_rewrites_loopback_hosts() { - assert_eq!( - docker_container_openshell_endpoint( - "https://localhost:8443", - HOST_OPENSHELL_INTERNAL, - DEFAULT_SERVER_PORT, - ), - "https://host.openshell.internal:17670/" - ); - assert_eq!( - docker_container_openshell_endpoint( - "http://127.0.0.1:8080", - HOST_OPENSHELL_INTERNAL, - DEFAULT_SERVER_PORT, - ), - "http://host.openshell.internal:17670/" - ); - assert_eq!( - docker_container_openshell_endpoint( - "https://gateway.internal:8443", - HOST_OPENSHELL_INTERNAL, - DEFAULT_SERVER_PORT, - ), - "https://host.openshell.internal:17670/" - ); -} - #[test] fn docker_bridge_gateway_ip_requires_ipv4_gateway() { let network = bollard::models::NetworkInspect { @@ -1109,13 +990,21 @@ fn docker_gateway_route_uses_host_gateway_for_docker_desktop() { ), DockerGatewayRoute::HostGateway ); - assert_eq!( - docker_extra_hosts(&DockerGatewayRoute::HostGateway), - vec![ - "host.docker.internal:host-gateway".to_string(), - "host.openshell.internal:host-gateway".to_string() - ] - ); +} + +#[test] +fn vm_backed_docker_daemon_uses_daemon_local_companion_transport() { + let desktop = SystemInfo { + operating_system: Some("Docker Desktop".to_string()), + ..Default::default() + }; + let native = SystemInfo { + operating_system: Some("Ubuntu 24.04".to_string()), + ..Default::default() + }; + + assert!(uses_host_gateway_alias(&desktop)); + assert!(!uses_host_gateway_alias(&native)); } #[test] @@ -1160,13 +1049,6 @@ fn docker_gateway_route_uses_host_gateway_for_colima() { ), DockerGatewayRoute::HostGateway ); - assert_eq!( - docker_extra_hosts(&DockerGatewayRoute::HostGateway), - vec![ - "host.docker.internal:host-gateway".to_string(), - "host.openshell.internal:host-gateway".to_string() - ] - ); } #[test] @@ -1251,16 +1133,8 @@ fn docker_gateway_route_uses_bridge_gateway_for_linux_docker() { route, DockerGatewayRoute::Bridge { bind_address: "172.18.0.1:17670".parse().unwrap(), - host_alias_ip: IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), } ); - assert_eq!( - docker_extra_hosts(&route), - vec![ - "host.docker.internal:172.18.0.1".to_string(), - "host.openshell.internal:172.18.0.1".to_string() - ] - ); } #[test] @@ -1300,15 +1174,21 @@ fn docker_gateway_route_prefers_configured_host_gateway_ip() { route, DockerGatewayRoute::Bridge { bind_address: "172.20.0.4:17670".parse().unwrap(), - host_alias_ip: IpAddr::V4(Ipv4Addr::new(172, 20, 0, 4)), } ); +} + +#[test] +fn docker_supervisor_alias_matches_the_trusted_gateway_route() { + assert_eq!( + docker_supervisor_host_alias(&DockerGatewayRoute::Bridge { + bind_address: "172.20.0.4:17670".parse().unwrap(), + }), + "172.20.0.4" + ); assert_eq!( - docker_extra_hosts(&route), - vec![ - "host.docker.internal:172.20.0.4".to_string(), - "host.openshell.internal:172.20.0.4".to_string() - ] + docker_supervisor_host_alias(&DockerGatewayRoute::HostGateway), + "host-gateway" ); } @@ -1383,13 +1263,13 @@ fn docker_resource_limits_applies_cpu_and_memory_limits() { } #[test] -fn docker_pids_limit_uses_runtime_default_when_omitted() { +fn docker_pids_limit_uses_driver_default_and_allows_runtime_inherit() { assert_eq!( - docker_pids_limit(std::num::NonZeroI64::new(2048)).unwrap(), - Some(2048) + docker_pids_limit(DEFAULT_SANDBOX_PIDS_LIMIT).unwrap(), + Some(DEFAULT_SANDBOX_PIDS_LIMIT) ); - assert_eq!(docker_pids_limit(None).unwrap(), None); - assert!(docker_pids_limit(std::num::NonZeroI64::new(-1)).is_err()); + assert_eq!(docker_pids_limit(0).unwrap(), None); + assert!(docker_pids_limit(-1).is_err()); } #[test] @@ -1399,109 +1279,51 @@ fn docker_compute_config_disables_bind_mounts_by_default() { } #[test] -fn container_create_body_omits_pids_limit_by_default() { +fn container_create_body_sets_driver_owned_pids_limit() { let body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); let host_config = body.host_config.expect("host config"); - assert_eq!(host_config.pids_limit, None); -} - -#[test] -fn container_create_body_emits_configured_positive_pids_limit() { - let mut config = runtime_config(); - config.sandbox_pids_limit = std::num::NonZeroI64::new(4096); - let body = build_container_create_body(&test_sandbox(), &config).unwrap(); - assert_eq!( - body.host_config.expect("host config").pids_limit, - Some(4096) - ); -} - -#[test] -fn build_environment_sets_docker_tls_paths() { - let env = build_environment(&test_sandbox(), &runtime_config()); - assert!(env.contains(&format!("OPENSHELL_TLS_CA={TLS_CA_MOUNT_PATH}"))); - assert!(env.contains(&format!("OPENSHELL_TLS_CERT={TLS_CERT_MOUNT_PATH}"))); - assert!(env.contains(&format!("OPENSHELL_TLS_KEY={TLS_KEY_MOUNT_PATH}"))); - assert!(env.contains(&"TEMPLATE_ENV=template".to_string())); - assert!(env.contains(&"SPEC_ENV=spec".to_string())); - assert!(env.contains(&format!( - "{}={}", - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY - ))); - let encoded = env - .iter() - .find_map(|entry| { - entry - .strip_prefix("OPENSHELL_MAIN_PROCESS_SPEC=") - .map(str::to_string) - }) - .expect("main-process transport"); - let main = openshell_core::sandbox_env::MainProcessConfig::decode(&encoded).unwrap(); - // An omitted command is forwarded empty; the supervisor resolves the default - // login shell against the sandbox image at startup. - assert!(main.command.is_empty()); - assert!(main.tty); -} - -#[test] -fn build_environment_keeps_network_capabilities_driver_controlled() { - let mut sandbox = test_sandbox(); - sandbox.spec.as_mut().unwrap().environment.insert( - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), - "spoofed".to_string(), - ); - let env = build_environment(&sandbox, &runtime_config()); - assert!(env.contains(&format!( - "{}={}", - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY - ))); - assert!(!env.iter().any(|entry| entry.ends_with("=spoofed"))); + assert_eq!(host_config.pids_limit, Some(DEFAULT_SANDBOX_PIDS_LIMIT)); } #[test] -fn build_environment_protects_oci_identity_metadata() { +fn docker_child_environment_strips_supervisor_control_keys() { let mut sandbox = test_sandbox(); let spec = sandbox.spec.as_mut().unwrap(); - for (key, value) in [ - (openshell_core::sandbox_env::OCI_IMAGE_USER, "spoofed"), - (openshell_core::sandbox_env::SANDBOX_UID, "9999"), - (openshell_core::sandbox_env::SANDBOX_GID, "9999"), + for key in [ + openshell_core::sandbox_env::ENDPOINT, + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME, + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + openshell_core::sandbox_env::OCI_IMAGE_USER, + openshell_core::sandbox_env::SANDBOX_TOKEN, + openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, ] { - spec.environment.insert(key.to_string(), value.to_string()); + spec.environment + .insert(key.to_string(), "spoofed".to_string()); } + spec.environment + .insert("PATH".to_string(), "/agent/bin".to_string()); - let env = build_environment_for_oci_user(&sandbox, &runtime_config(), "app:staff"); + let env = docker_child_environment(&sandbox); - assert!(env.contains(&format!( - "{}=app:staff", - openshell_core::sandbox_env::OCI_IMAGE_USER - ))); - assert!(env.contains(&format!("{}=", openshell_core::sandbox_env::SANDBOX_UID))); - assert!(env.contains(&format!("{}=", openshell_core::sandbox_env::SANDBOX_GID))); - assert!(!env.iter().any(|entry| entry.ends_with("=spoofed"))); - assert!(!env.iter().any(|entry| entry.ends_with("=9999"))); + assert_eq!(env.get("PATH").map(String::as_str), Some("/agent/bin")); + assert!(env.contains_key("TEMPLATE_ENV")); + assert!(env.contains_key("SPEC_ENV")); + assert!(!env.values().any(|value| value == "spoofed")); } #[test] -fn build_environment_strips_gateway_tls_server_name() { - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.environment.insert( - openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME.to_string(), - "evil.attacker.example.com".to_string(), - ); - - let env = build_environment(&sandbox, &runtime_config()); +fn boundary_environment_contains_only_driver_owned_values() { + let env = build_boundary_environment(&test_sandbox(), &runtime_config()); + assert_eq!(env.len(), 2); assert!( - !env.iter().any(|entry| entry.starts_with(&format!( - "{}=", - openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME - ))), - "GATEWAY_TLS_SERVER_NAME must be stripped from the supervisor environment" + env.iter() + .any(|entry| entry.starts_with("OPENSHELL_LOG_LEVEL=")) ); + assert!(env.iter().any(|entry| entry.starts_with(&format!( + "{}=", + openshell_core::sandbox_env::TELEMETRY_ENABLED + )))); } #[test] @@ -1519,111 +1341,343 @@ fn container_creation_uses_inspected_immutable_image() { &DockerSandboxDriverConfig::default(), None, &metadata, + &test_workload_identity(), ) .unwrap(); assert_eq!(body.image.as_deref(), Some("sha256:immutable")); - assert_eq!(body.user.as_deref(), Some("0")); + assert_eq!(body.user.as_deref(), Some("1234:1235")); assert_eq!(body.working_dir.as_deref(), Some("/")); + assert_eq!( + body.labels + .as_ref() + .and_then(|labels| labels.get(LABEL_ISOLATION_TOPOLOGY)) + .map(String::as_str), + Some(LABEL_ISOLATION_TOPOLOGY_CAPABILITY_FREE) + ); + assert_eq!( + body.labels + .as_ref() + .and_then(|labels| labels.get(LABEL_ISOLATION_ROLE)) + .map(String::as_str), + Some(LABEL_ISOLATION_ROLE_SANDBOX) + ); assert_eq!( body.cmd.as_deref(), - Some(&["--workdir".to_string(), "/workspace/project".to_string()][..]) + Some( + &[ + "--bootstrap".to_string(), + BOUNDARY_CONFIG_MOUNT_PATH.to_string(), + ][..] + ) ); - assert!(body.env.unwrap().contains(&format!( - "{}=1234:1235", - openshell_core::sandbox_env::OCI_IMAGE_USER - ))); + assert!(body.env.unwrap().iter().all(|entry| { + !entry.starts_with(&format!("{}=", openshell_core::sandbox_env::OCI_IMAGE_USER)) + })); + let host = body.host_config.unwrap(); + assert_eq!(host.cap_add, None); + assert_eq!(host.cap_drop, Some(vec!["ALL".to_string()])); + assert_eq!(host.group_add, Some(vec!["1236".to_string()])); + assert_eq!( + host.security_opt, + Some(vec!["no-new-privileges:true".to_string()]) + ); + assert_eq!(host.network_mode.as_deref(), Some("none")); + assert_eq!(host.dns, Some(vec!["127.0.0.53".to_string()])); } #[test] -fn container_creation_rejects_invalid_oci_working_dir() { - let metadata = DockerImageMetadata { - id: "sha256:immutable".to_string(), - user: "1234:1235".to_string(), - working_dir: "relative/workspace".to_string(), - volumes: Vec::new(), +fn docker_outer_fence_accepts_network_none_without_attachments() { + let inspected = bollard::models::ContainerInspectResponse { + host_config: Some(HostConfig { + network_mode: Some("none".to_string()), + ..Default::default() + }), + network_settings: Some(bollard::models::NetworkSettings { + networks: Some(HashMap::from([( + "none".to_string(), + bollard::models::EndpointSettings::default(), + )])), + ..Default::default() + }), + ..Default::default() }; - let err = build_container_create_body_for_image( - &test_sandbox(), - &runtime_config(), - &DockerSandboxDriverConfig::default(), - None, - &metadata, - ) - .unwrap_err(); - assert_eq!(err.code(), tonic::Code::FailedPrecondition); - assert!(err.message().contains("must be an absolute container path")); + assert!(validate_docker_outer_fence(&inspected).is_ok()); } #[test] -fn container_creation_rejects_openshell_control_path_working_dir() { - let metadata = DockerImageMetadata { - id: "sha256:immutable".to_string(), - user: "1234:1235".to_string(), - working_dir: "/opt/openshell/bin/project".to_string(), - volumes: Vec::new(), +fn docker_outer_fence_rejects_network_mode_or_attached_network_drift() { + let bridge_mode = bollard::models::ContainerInspectResponse { + host_config: Some(HostConfig { + network_mode: Some("bridge".to_string()), + ..Default::default() + }), + ..Default::default() + }; + let attached_network = bollard::models::ContainerInspectResponse { + host_config: Some(HostConfig { + network_mode: Some("none".to_string()), + ..Default::default() + }), + network_settings: Some(bollard::models::NetworkSettings { + networks: Some(HashMap::from([( + "unexpected".to_string(), + bollard::models::EndpointSettings::default(), + )])), + ..Default::default() + }), + ..Default::default() }; - let err = build_container_create_body_for_image( - &test_sandbox(), - &runtime_config(), - &DockerSandboxDriverConfig::default(), - None, - &metadata, - ) - .unwrap_err(); - assert_eq!(err.code(), tonic::Code::FailedPrecondition); - assert!(err.message().contains("OpenShell control path")); + assert!(validate_docker_outer_fence(&bridge_mode).is_err()); + assert!(validate_docker_outer_fence(&attached_network).is_err()); } #[test] -fn container_creation_rejects_image_volume_that_masks_working_dir() { - let sandbox = test_sandbox(); - let metadata = DockerImageMetadata { - id: "sha256:immutable".to_string(), - user: "1234:1235".to_string(), - working_dir: "/workspace/project".to_string(), - volumes: vec!["/workspace".to_string()], - }; - - let error = build_container_create_body_for_image( - &sandbox, - &runtime_config(), - &DockerSandboxDriverConfig::default(), - None, - &metadata, +fn sandbox_bundle_prepares_only_the_driver_managed_workspace() { + let identity = test_workload_identity(); + let default_archive = docker_sandbox_bundle_archive( + b"sandbox-binary", + b"{}", + DockerSandboxTls { + certificate: b"server-cert", + private_key: b"server-key", + client_ca: b"client-ca", + }, + &identity, + driver_mounts::DEFAULT_WORKSPACE_ROOT, ) - .unwrap_err(); - - assert!( - error - .message() - .contains("masks OCI WorkingDir '/workspace/project'") + .unwrap(); + let mut archive = tar::Archive::new(default_archive.as_slice()); + let sandbox_entry = archive + .entries() + .unwrap() + .map(Result::unwrap) + .find(|entry| entry.path().unwrap().as_ref() == Path::new("sandbox")) + .expect("managed /sandbox entry"); + assert!(sandbox_entry.header().entry_type().is_dir()); + assert_eq!(sandbox_entry.header().mode().unwrap(), 0o700); + assert_eq!( + sandbox_entry.header().uid().unwrap(), + u64::from(identity.uid) + ); + assert_eq!( + sandbox_entry.header().gid().unwrap(), + u64::from(identity.gid) ); -} -#[test] -fn container_creation_rejects_image_volume_over_configured_ssh_socket() { + let image_archive = docker_sandbox_bundle_archive( + b"sandbox-binary", + b"{}", + DockerSandboxTls { + certificate: b"server-cert", + private_key: b"server-key", + client_ca: b"client-ca", + }, + &identity, + "/workspace/project", + ) + .unwrap(); + let mut archive = tar::Archive::new(image_archive.as_slice()); + assert!( + archive + .entries() + .unwrap() + .map(Result::unwrap) + .all(|entry| { entry.path().unwrap().as_ref() != Path::new("workspace/project") }) + ); +} + +#[test] +fn sandbox_bundle_stages_private_mutual_tls_material() { + let identity = test_workload_identity(); + let archive = docker_sandbox_bundle_archive( + b"sandbox-binary", + b"{}", + DockerSandboxTls { + certificate: b"server-cert", + private_key: b"server-key", + client_ca: b"client-ca", + }, + &identity, + driver_mounts::DEFAULT_WORKSPACE_ROOT, + ) + .unwrap(); + let mut archive = tar::Archive::new(archive.as_slice()); + let entries = archive + .entries() + .unwrap() + .map(Result::unwrap) + .filter_map(|entry| { + let path = entry.path().ok()?.into_owned(); + Some(( + path, + ( + entry.header().mode().ok()?, + entry.header().uid().ok()?, + entry.header().gid().ok()?, + ), + )) + }) + .collect::>(); + for path in [ + ".openshell/channel/sandbox/server.crt", + ".openshell/channel/sandbox/server.key", + ".openshell/channel/sandbox/client-ca.crt", + ] { + assert_eq!( + entries.get(Path::new(path)), + Some(&(0o600, u64::from(identity.uid), u64::from(identity.gid))) + ); + } + assert_eq!( + entries.get(Path::new(".openshell/runtime/openshell-sandbox")), + Some(&(0o555, 0, 0)), + "the trusted sandbox executable must not be writable by the workload" + ); + assert_eq!( + entries.get(Path::new(".openshell/channel")), + Some(&(0o755, 0, 0)), + "the workload must not be able to replace the supervisor secret directory" + ); + assert_eq!( + entries.get(Path::new(".openshell/channel/sandbox")), + Some(&(0o711, u64::from(identity.uid), u64::from(identity.gid))), + "the supervisor must be able to traverse to the authenticated socket without reading sandbox secrets" + ); +} + +#[test] +fn docker_identity_resolution_uses_pinned_image_accounts_and_exact_groups() { + let sandbox = test_sandbox(); + let image = DockerImageMetadata { + id: "sha256:image".to_string(), + user: "agent".to_string(), + working_dir: "/sandbox".to_string(), + volumes: Vec::new(), + }; + let resolved = resolve_docker_identity_from_accounts( + &sandbox, + &image, + b"root:x:0:0:root:/root:/bin/sh\nagent:x:10001:10002::/sandbox:/bin/sh\n", + b"root:x:0:\nagent:x:10002:\nrender:x:10003:agent\n", + ) + .unwrap(); + + assert_eq!(resolved.uid, 10001); + assert_eq!(resolved.gid, 10002); + assert_eq!(resolved.supplementary_gids, vec![10003]); + assert_eq!(resolved.source, "image"); + assert_eq!(resolved.resource_digest, "sha256:image"); +} + +#[test] +fn docker_identity_resolution_honors_policy_selectors_and_rejects_root() { + let mut sandbox = test_sandbox(); + sandbox.spec.as_mut().unwrap().workload_identity = Some(WorkloadIdentityRequest { + user: "10001".to_string(), + group: "workers".to_string(), + }); + let image = DockerImageMetadata { + id: "sha256:image".to_string(), + user: String::new(), + working_dir: "/sandbox".to_string(), + volumes: Vec::new(), + }; + let resolved = resolve_docker_identity_from_accounts( + &sandbox, + &image, + b"agent:x:10001:10002::/sandbox:/bin/sh\n", + b"workers:x:10004:agent\n", + ) + .unwrap(); + assert_eq!((resolved.uid, resolved.gid), (10001, 10004)); + assert_eq!(resolved.source, "policy"); + + sandbox.spec.as_mut().unwrap().workload_identity = Some(WorkloadIdentityRequest { + user: "root".to_string(), + group: "root".to_string(), + }); + let error = resolve_docker_identity_from_accounts( + &sandbox, + &image, + b"root:x:0:0:root:/root:/bin/sh\n", + b"root:x:0:\n", + ) + .unwrap_err(); + assert!(error.message().contains("UID or GID zero")); +} + +#[test] +fn container_creation_rejects_invalid_oci_working_dir() { let metadata = DockerImageMetadata { id: "sha256:immutable".to_string(), user: "1234:1235".to_string(), - working_dir: "/workspace".to_string(), - volumes: vec!["/custom-runtime".to_string()], + working_dir: "relative/workspace".to_string(), + volumes: Vec::new(), }; - let mut config = runtime_config(); - config.ssh_socket_path = "/custom-runtime/ssh.sock".to_string(); + let err = build_container_create_body_for_image( + &test_sandbox(), + &runtime_config(), + &DockerSandboxDriverConfig::default(), + None, + &metadata, + &test_workload_identity(), + ) + .unwrap_err(); - let error = build_container_create_body_for_image( + assert_eq!(err.code(), tonic::Code::FailedPrecondition); + assert!(err.message().contains("must be an absolute container path")); +} + +#[test] +fn container_creation_rejects_openshell_control_path_working_dir() { + let metadata = DockerImageMetadata { + id: "sha256:immutable".to_string(), + user: "1234:1235".to_string(), + working_dir: "/opt/openshell/bin/project".to_string(), + volumes: Vec::new(), + }; + let err = build_container_create_body_for_image( &test_sandbox(), - &config, + &runtime_config(), &DockerSandboxDriverConfig::default(), None, &metadata, + &test_workload_identity(), ) .unwrap_err(); - assert!(error.message().contains("OpenShell control path")); + assert_eq!(err.code(), tonic::Code::FailedPrecondition); + assert!(err.message().contains("OpenShell control path")); +} + +#[test] +fn container_creation_rejects_image_volume_that_masks_working_dir() { + let sandbox = test_sandbox(); + let metadata = DockerImageMetadata { + id: "sha256:immutable".to_string(), + user: "1234:1235".to_string(), + working_dir: "/workspace/project".to_string(), + volumes: vec!["/workspace".to_string()], + }; + + let error = build_container_create_body_for_image( + &sandbox, + &runtime_config(), + &DockerSandboxDriverConfig::default(), + None, + &metadata, + &test_workload_identity(), + ) + .unwrap_err(); + + assert!( + error + .message() + .contains("masks OCI WorkingDir '/workspace/project'") + ); } #[test] @@ -1644,6 +1698,7 @@ fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts( &root_mount, None, &metadata, + &test_workload_identity(), ) .unwrap_err(); assert!( @@ -1666,6 +1721,7 @@ fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts( &ancestor_mount, None, &nested_metadata, + &test_workload_identity(), ) .unwrap_err(); assert!( @@ -1683,6 +1739,7 @@ fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts( &nested_mount, None, &metadata, + &test_workload_identity(), ) .expect("nested workspace mounts remain supported"); @@ -1697,84 +1754,15 @@ fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts( &compatibility_path_mount, None, &metadata, + &test_workload_identity(), ) .expect("/sandbox remains mountable when the inspected workspace is elsewhere"); } #[test] -fn build_environment_keeps_path_driver_controlled() { - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.environment - .insert("PATH".to_string(), "/malicious/spec/bin".to_string()); - spec.template - .as_mut() - .unwrap() - .environment - .insert("PATH".to_string(), "/malicious/template/bin".to_string()); - - let env = build_environment(&sandbox, &runtime_config()); - let path_entries = env - .iter() - .filter(|entry| entry.starts_with("PATH=")) - .collect::>(); - - let expected_path = format!("PATH={SUPERVISOR_PATH}"); - assert_eq!(path_entries.len(), 1); - assert_eq!(path_entries[0], &expected_path); -} - -#[test] -fn build_environment_keeps_telemetry_toggle_driver_controlled() { - let _guard = ENV_LOCK.lock().unwrap(); - temp_env::with_vars( - [( - openshell_core::sandbox_env::TELEMETRY_ENABLED, - Some("false"), - )], - || { - let mut sandbox = test_sandbox(); - sandbox.spec.as_mut().unwrap().environment.insert( - openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), - "true".to_string(), - ); - - let env = build_environment(&sandbox, &runtime_config()); - let telemetry_entries = env - .iter() - .filter(|entry| { - entry.starts_with(&format!( - "{}=", - openshell_core::sandbox_env::TELEMETRY_ENABLED - )) - }) - .collect::>(); - - assert_eq!(telemetry_entries.len(), 1); - assert_eq!( - telemetry_entries[0], - &format!("{}=false", openshell_core::sandbox_env::TELEMETRY_ENABLED) - ); - }, - ); -} - -#[test] -fn build_binds_uses_docker_tls_directory() { - let binds = build_binds(&test_sandbox(), &runtime_config()).unwrap(); - let targets = binds - .iter() - .filter_map(|bind| bind.split(':').nth(1).map(String::from)) - .collect::>(); - assert!(targets.contains(&SUPERVISOR_MOUNT_PATH.to_string())); - assert!(targets.contains(&TLS_CA_MOUNT_PATH.to_string())); - assert!(targets.contains(&TLS_CERT_MOUNT_PATH.to_string())); - assert!(targets.contains(&TLS_KEY_MOUNT_PATH.to_string())); - assert!( - targets - .iter() - .all(|target| target.starts_with(TLS_MOUNT_DIR) || target == SUPERVISOR_MOUNT_PATH) - ); +fn build_binds_does_not_expose_host_runtime_material() { + let binds = build_binds(&test_sandbox(), &runtime_config()); + assert!(binds.is_empty()); } #[test] @@ -1807,7 +1795,7 @@ fn build_container_create_body_includes_driver_config_mounts() { .mounts .expect("driver config mounts should be set"); - assert_eq!(mounts.len(), 2); + assert_eq!(mounts.len(), 3); assert_eq!(mounts[0].typ, Some(MountTypeEnum::VOLUME)); assert_eq!(mounts[0].source.as_deref(), Some("work-nfs")); assert_eq!(mounts[0].target.as_deref(), Some("/sandbox/work")); @@ -1821,6 +1809,9 @@ fn build_container_create_body_includes_driver_config_mounts() { ); assert_eq!(mounts[1].typ, Some(MountTypeEnum::TMPFS)); assert_eq!(mounts[1].target.as_deref(), Some("/sandbox/cache")); + assert_eq!(mounts[2].typ, Some(MountTypeEnum::VOLUME)); + assert_eq!(mounts[2].target.as_deref(), Some(BOUNDARY_MOUNT_PATH)); + assert_eq!(mounts[2].read_only, Some(false)); assert_eq!( mounts[1] .tmpfs_options @@ -2246,36 +2237,6 @@ fn driver_config_rejects_reserved_mount_targets() { assert!(err.message().contains("reserved OpenShell path")); } -#[test] -fn driver_config_rejects_mount_over_configured_ssh_socket() { - let mount_config: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ - "mounts": [{ - "type": "tmpfs", - "target": "/custom-runtime" - }] - })) - .unwrap(); - let metadata = DockerImageMetadata { - id: "sha256:immutable".to_string(), - user: "1234:1235".to_string(), - working_dir: "/workspace".to_string(), - volumes: Vec::new(), - }; - let mut config = runtime_config(); - config.ssh_socket_path = "/custom-runtime/ssh.sock".to_string(); - - let error = build_container_create_body_for_image( - &test_sandbox(), - &config, - &mount_config, - None, - &metadata, - ) - .unwrap_err(); - - assert!(error.message().contains("OpenShell control path")); -} - #[test] fn docker_local_volume_with_bind_option_is_bind_backed() { let volume = inspected_volume( @@ -2328,72 +2289,6 @@ fn docker_nonlocal_volume_with_bind_option_is_not_bind_backed() { assert!(!docker_volume_is_bind_backed(&volume)); } -#[test] -fn build_environment_uses_token_file_without_raw_token_env() { - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.sandbox_token = "secret.jwt.value".to_string(); - spec.environment.insert( - openshell_core::sandbox_env::SANDBOX_TOKEN.to_string(), - "user-provided-token".to_string(), - ); - - let env = build_environment(&sandbox, &runtime_config()); - - assert!(!env.iter().any(|entry| { - entry.starts_with(&format!("{}=", openshell_core::sandbox_env::SANDBOX_TOKEN)) - })); - assert!(env.contains(&format!( - "{}={SANDBOX_TOKEN_MOUNT_PATH}", - openshell_core::sandbox_env::SANDBOX_TOKEN_FILE - ))); -} - -#[test] -fn docker_container_projects_proxy_and_spiffe_without_credential_metadata() { - let mut config = runtime_config(); - config.upstream_proxy = UpstreamProxyConfig { - https_proxy: Some("https://proxy.example:8443".to_string()), - no_proxy: Some(".svc".to_string()), - proxy_auth_file: Some(PathBuf::from("/run/secrets/proxy-auth")), - proxy_auth_allow_insecure: None, - proxy_connect_by_hostname: Some(true), - }; - config.provider_spiffe_workload_api_socket = Some(PathBuf::from("/run/spire/agent.sock")); - let body = build_container_create_body(&test_sandbox(), &config).unwrap(); - let command = body.cmd.unwrap(); - assert!( - command - .windows(2) - .any(|args| args == ["--upstream-proxy", "https://proxy.example:8443"]) - ); - assert!( - command - .windows(2) - .any(|args| args == ["--upstream-proxy-auth-file", UPSTREAM_PROXY_AUTH_MOUNT_PATH]) - ); - assert!( - command - .windows(2) - .any(|args| args == ["--upstream-no-proxy", ".svc"]) - ); - assert!(command.contains(&"--upstream-proxy-connect-by-hostname".to_string())); - let binds = body.host_config.unwrap().binds.unwrap(); - assert!( - binds - .iter() - .any(|bind| bind.contains(UPSTREAM_PROXY_AUTH_MOUNT_PATH)) - ); - assert!(binds.contains(&format!( - "/run/spire:{PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR}:ro" - ))); - assert!(binds.iter().all(|bind| !bind.contains("rbind"))); - let env = body.env.unwrap(); - assert!(env.iter().any(|entry| entry - == "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET=/spiffe-workload-api/agent.sock")); - assert!(!env.iter().any(|entry| entry.contains("proxy-auth"))); -} - #[test] fn managed_container_label_filters_include_gateway_namespace() { let filters = @@ -2402,20 +2297,26 @@ fn managed_container_label_filters_include_gateway_namespace() { assert!(labels.contains(&format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}"))); assert!(labels.contains(&format!("{LABEL_SANDBOX_NAMESPACE}=tenant-a"))); + assert!(labels.contains(&format!( + "{LABEL_ISOLATION_ROLE}={LABEL_ISOLATION_ROLE_SANDBOX}" + ))); assert!(labels.contains(&format!("{LABEL_SANDBOX_ID}=sbx-123"))); } #[test] -fn build_container_create_body_replaces_inherited_cmd_with_workspace_arg() { +fn build_container_create_body_replaces_inherited_cmd_with_sandbox_bootstrap() { let create_body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); assert_eq!( create_body.entrypoint, - Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]) + Some(vec![SANDBOX_BINARY_PATH.to_string()]) ); assert_eq!( create_body.cmd, - Some(vec!["--workdir".to_string(), "/sandbox".to_string()]) + Some(vec![ + "--bootstrap".to_string(), + BOUNDARY_CONFIG_MOUNT_PATH.to_string(), + ]) ); assert_eq!( create_body @@ -2431,27 +2332,11 @@ fn build_container_create_body_replaces_inherited_cmd_with_workspace_arg() { ); assert_eq!( host_config.security_opt.as_ref(), - Some(&vec!["apparmor=unconfined".to_string()]) - ); - assert_eq!( - host_config.network_mode.as_deref(), - Some(DEFAULT_DOCKER_NETWORK_NAME) - ); - assert_eq!( - host_config.extra_hosts.as_ref(), - Some(&vec![ - "host.docker.internal:172.18.0.1".to_string(), - "host.openshell.internal:172.18.0.1".to_string() - ]) - ); - assert_eq!( - create_body - .networking_config - .as_ref() - .and_then(|config| config.endpoints_config.as_ref()) - .and_then(|endpoints| endpoints.get(DEFAULT_DOCKER_NETWORK_NAME)), - Some(&EndpointSettings::default()) + Some(&vec!["no-new-privileges:true".to_string()]) ); + assert_eq!(host_config.network_mode.as_deref(), Some("none")); + assert_eq!(host_config.extra_hosts, None); + assert!(create_body.networking_config.is_none()); } #[test] @@ -2900,23 +2785,17 @@ fn require_sandbox_identifier_rejects_when_id_and_name_are_empty() { } #[test] -fn build_container_create_body_uses_bridge_network() { +fn build_container_create_body_disables_docker_networking() { let create_body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); let host_config = create_body.host_config.expect("host_config is populated"); assert_eq!( host_config.network_mode, - Some(DEFAULT_DOCKER_NETWORK_NAME.to_string()), - "sandbox should join the driver-managed bridge network" - ); - assert_eq!( - host_config.extra_hosts, - Some(vec![ - "host.docker.internal:172.18.0.1".to_string(), - "host.openshell.internal:172.18.0.1".to_string() - ]), - "sandbox should expose stable host aliases for gateway callbacks" + Some("none".to_string()), + "the sandbox must not receive direct Docker networking" ); + assert_eq!(host_config.extra_hosts, None); + assert_eq!(host_config.dns, Some(vec!["127.0.0.53".to_string()])); } #[test] @@ -2925,10 +2804,10 @@ fn build_container_create_body_uses_runtime_namespace_label() { // runtime config, not from `DriverSandbox.namespace`. The gateway // does not populate `DriverSandbox.namespace`, so a container created // with that empty value would not match subsequent list/get/find - // queries (which filter on `config.sandbox_label`), leaking + // queries (which filter on `config.sandbox_namespace`), leaking // sandboxes that the driver itself cannot observe. let mut config = runtime_config(); - config.sandbox_label = "tenant-a".to_string(); + config.sandbox_namespace = "tenant-a".to_string(); let mut sandbox = test_sandbox(); sandbox.namespace = "ignored-by-driver".to_string(); @@ -3109,17 +2988,8 @@ fn pending_sandbox_snapshot_uses_docker_namespace_and_starting_condition() { assert_eq!(snapshot.name, "demo"); assert_eq!(snapshot.namespace, "docker-dev"); assert!(snapshot.spec.is_none()); - let pending = pending_map(&[&snapshot]); - assert_eq!( - resolve_pending_id(&pending, "sbx-123", "") - .unwrap() - .as_deref(), - Some("sbx-123") - ); - assert_eq!( - resolve_pending_id(&pending, "", "demo").unwrap().as_deref(), - Some("sbx-123") - ); + assert!(pending_sandbox_matches(&snapshot, "sbx-123", "")); + assert!(pending_sandbox_matches(&snapshot, "", "demo")); let status = snapshot.status.expect("status"); assert!(!status.deleting); @@ -3131,16 +3001,6 @@ fn pending_sandbox_snapshot_uses_docker_namespace_and_starting_condition() { assert_eq!(status.conditions[0].message, "Docker container is starting"); } -#[test] -fn validate_linux_elf_binary_rejects_non_elf_files() { - let tempdir = TempDir::new().unwrap(); - let path = tempdir.path().join("openshell-sandbox"); - fs::write(&path, b"not-elf").unwrap(); - - let err = validate_linux_elf_binary(&path).unwrap_err(); - assert!(err.contains("Linux ELF executable")); -} - #[test] fn docker_guest_tls_paths_require_all_files_for_https() { let tempdir = TempDir::new().unwrap(); @@ -3156,22 +3016,6 @@ fn docker_guest_tls_paths_require_all_files_for_https() { assert!(err.to_string().contains("guest_tls_cert")); } -#[test] -fn linux_supervisor_candidates_follow_daemon_arch() { - assert_eq!( - linux_supervisor_candidates("amd64"), - vec![PathBuf::from( - "target/x86_64-unknown-linux-gnu/release/openshell-sandbox", - )] - ); - assert_eq!( - linux_supervisor_candidates("arm64"), - vec![PathBuf::from( - "target/aarch64-unknown-linux-gnu/release/openshell-sandbox", - )] - ); -} - #[test] fn container_name_preserves_id_suffix_for_long_names() { // Names up to 253 chars are permitted by the gRPC layer. The id @@ -3258,32 +3102,6 @@ fn docker_guest_tls_paths_allows_plain_http_without_tls_flags() { assert!(result.is_none()); } -#[test] -fn docker_automatic_tls_detection_is_fail_closed_for_partial_bundles() { - for mask in 0_u8..8 { - let config = DockerComputeConfig { - guest_tls_ca: (mask & 1 != 0).then(|| PathBuf::from("/tmp/ca.pem")), - guest_tls_cert: (mask & 2 != 0).then(|| PathBuf::from("/tmp/cert.pem")), - guest_tls_key: (mask & 4 != 0).then(|| PathBuf::from("/tmp/key.pem")), - ..Default::default() - }; - assert_eq!( - docker_guest_tls_configured(&config), - mask != 0, - "TLS presence mask {mask:03b}" - ); - - if mask != 0 && mask != 7 { - let mut inferred = config; - inferred.grpc_endpoint = "https://host.openshell.internal:8080".to_string(); - assert!( - docker_guest_tls_paths(&inferred).is_err(), - "partial TLS presence mask {mask:03b} must fail" - ); - } - } -} - #[test] fn default_docker_supervisor_image_uses_nvidia_ghcr_repo() { let image = openshell_core::config::default_supervisor_image(); @@ -3293,36 +3111,6 @@ fn default_docker_supervisor_image_uses_nvidia_ghcr_repo() { ); } -#[test] -fn configured_supervisor_image_takes_precedence_over_local_binaries() { - let tempdir = TempDir::new().unwrap(); - let bin_dir = tempdir.path().join("bin"); - fs::create_dir_all(&bin_dir).unwrap(); - let current_exe = bin_dir.join("openshell-gateway"); - let sibling = bin_dir.join("openshell-sandbox"); - fs::write(¤t_exe, b"gateway").unwrap(); - fs::write(&sibling, b"\x7fELFsibling").unwrap(); - - let local_build = tempdir.path().join("target/openshell-sandbox"); - fs::create_dir_all(local_build.parent().unwrap()).unwrap(); - fs::write(&local_build, b"\x7fELFlocal").unwrap(); - - let source = resolve_supervisor_bin_source( - &DockerComputeConfig { - supervisor_image: Some("example.com/openshell/supervisor:test".to_string()), - ..Default::default() - }, - Some(¤t_exe), - &[local_build], - ) - .unwrap(); - - assert_eq!( - source, - SupervisorBinSource::Image("example.com/openshell/supervisor:test".to_string()) - ); -} - #[test] fn docker_supervisor_image_tag_prefers_explicit_build_tags() { use openshell_core::config::resolve_supervisor_image_tag; @@ -3367,63 +3155,6 @@ fn docker_supervisor_image_refreshes_mutable_tags_only() { )); } -#[test] -fn supervisor_cache_path_namespaces_by_digest_under_openshell_data_dir() { - let base = PathBuf::from("/var/cache/share"); - let path = supervisor_cache_path_with_base( - &base, - "docker-supervisor", - "sha256:abc123deadbeef0123456789cafe0123456789fe", - ); - - assert_eq!( - path, - PathBuf::from( - "/var/cache/share/openshell/docker-supervisor/sha256-abc123deadbeef0123456789cafe0123456789fe/openshell-sandbox", - ), - ); -} - -#[test] -fn supervisor_cache_path_isolates_different_digests() { - let base = PathBuf::from("/data"); - let left = supervisor_cache_path_with_base(&base, "docker-supervisor", "sha256:aaaaaaaa"); - let right = supervisor_cache_path_with_base(&base, "docker-supervisor", "sha256:bbbbbbbb"); - assert_ne!( - left.parent().unwrap(), - right.parent().unwrap(), - "digest-keyed directories must differ so rollouts are isolated", - ); -} - -#[test] -fn write_cache_binary_atomic_materializes_file_with_executable_mode() { - let tempdir = TempDir::new().unwrap(); - let target = tempdir.path().join("nested").join("openshell-sandbox"); - fs::create_dir_all(target.parent().unwrap()).unwrap(); - - write_cache_binary_atomic(&target, b"\x7fELFpayload").unwrap(); - - assert!(target.is_file()); - assert_eq!(fs::read(&target).unwrap(), b"\x7fELFpayload"); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let mode = fs::metadata(&target).unwrap().permissions().mode() & 0o777; - assert_eq!(mode, 0o755, "expected 0755, got {mode:04o}"); - } -} - -#[test] -fn write_cache_binary_atomic_overwrites_existing_file() { - let tempdir = TempDir::new().unwrap(); - let target = tempdir.path().join("openshell-sandbox"); - fs::write(&target, b"stale").unwrap(); - - write_cache_binary_atomic(&target, b"\x7fELFfresh").unwrap(); - assert_eq!(fs::read(&target).unwrap(), b"\x7fELFfresh"); -} - #[test] fn temp_extract_container_names_are_unique_per_call() { let first = temp_extract_container_name(); @@ -3496,6 +3227,13 @@ fn lifecycle_fence_rejects_polled_exit_from_before_restart() { fences.finish_start("sandbox-1"); assert!(!fences.start_in_progress("sandbox-1")); + fences.request_stop("sandbox-1", "demo"); + assert!(fences.stop_requested("sandbox-1", "")); + assert!(fences.stop_requested("", "demo")); + fences.clear_stop("sandbox-1", "demo"); + assert!(!fences.stop_requested("sandbox-1", "demo")); + fences.request_stop("sandbox-1", "demo"); + fences.record_previous_exit("sandbox-1", Some("2026-08-12T16:39:13Z")); assert_eq!( fences.previous_exit("sandbox-1").as_deref(), @@ -3530,8 +3268,10 @@ fn lifecycle_fence_rejects_polled_exit_from_before_restart() { Some(&new_exit), )); - fences.remove("sandbox-1"); + fences.remove("sandbox-1", "demo"); assert!(fences.previous_exit("sandbox-1").is_none()); + assert!(!fences.stop_requested("sandbox-1", "")); + assert!(!fences.stop_requested("", "demo")); } fn exited_sandbox_with_ready_reason(reason: &str) -> DriverSandbox { @@ -3568,15 +3308,6 @@ fn ready_reason(sandbox: &DriverSandbox) -> &str { .expect("Ready condition present") } -fn ready_message(sandbox: &DriverSandbox) -> &str { - sandbox - .status - .as_ref() - .and_then(|status| status.conditions.iter().find(|c| c.r#type == "Ready")) - .map(|c| c.message.as_str()) - .expect("Ready condition present") -} - #[test] fn docker_signal_kill_reclassified_as_runtime_restart() { // 137 (128+SIGKILL) and 143 (128+SIGTERM) mark an external termination — @@ -3614,24 +3345,6 @@ fn docker_ordinary_exit_stays_terminal() { assert_eq!(ready_reason(&sandbox), CONDITION_EXITED); } -#[test] -fn docker_workspace_validation_exit_is_reported_explicitly() { - let mut sandbox = exited_sandbox_with_ready_reason(CONDITION_EXITED); - let state = ContainerState { - status: Some(ContainerStateStatusEnum::EXITED), - exit_code: Some(i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)), - ..Default::default() - }; - - apply_docker_exit_classification(&mut sandbox, &state); - - assert_eq!( - ready_reason(&sandbox), - CONDITION_WORKSPACE_VALIDATION_FAILED - ); - assert!(ready_message(&sandbox).contains("WorkingDir")); -} - #[test] fn docker_oom_kill_stays_terminal_despite_137() { // An OOM kill reports exit 137 but must NOT be treated as a recoverable @@ -3647,406 +3360,17 @@ fn docker_oom_kill_stays_terminal_despite_137() { assert_eq!(ready_reason(&sandbox), CONDITION_EXITED); } -/// Minimal pending-map entry. Only the identity fields matter for lookup -/// resolution, so the spec and status are left empty on purpose. -fn pending_sandbox(id: &str, name: &str, workspace: &str) -> DriverSandbox { - DriverSandbox { - id: id.to_string(), - name: name.to_string(), - namespace: String::new(), - spec: None, - status: None, - workspace: workspace.to_string(), - } -} - -fn pending_map(sandboxes: &[&DriverSandbox]) -> HashMap { - sandboxes - .iter() - .map(|sandbox| { - ( - sandbox.id.clone(), - PendingSandboxRecord { - sandbox: (*sandbox).clone(), - task: None, - }, - ) - }) - .collect() -} - -async fn driver_with_pending(sandboxes: &[&DriverSandbox]) -> DockerComputeDriver { - let driver = test_driver_with_config(runtime_config()); - for sandbox in sandboxes { - driver - .reserve_pending_sandbox(sandbox) - .await - .expect("reserving a distinct sandbox must succeed"); - } - driver -} - -fn pending_ids(pending: &HashMap) -> Vec { - let mut ids: Vec = pending.keys().cloned().collect(); - ids.sort(); - ids -} - -#[test] -fn resolve_pending_id_prefers_sandbox_id_over_sandbox_name() { - // The id is authoritative. A stale or mismatched name travelling in the - // same request must not change which record is resolved. - let alpha = pending_sandbox("sbx-alpha", "demo", "alpha"); - let pending = pending_map(&[&alpha]); - - assert_eq!( - resolve_pending_id(&pending, "sbx-alpha", "stale-name") - .unwrap() - .as_deref(), - Some("sbx-alpha") - ); -} - -#[test] -fn resolve_pending_id_ignores_the_name_when_the_id_is_not_pending() { - // Regression for the `id OR name` match. `demo` exists in two workspaces: - // the beta copy is still provisioning, the alpha copy is already running. - // Deleting the alpha copy sends alpha's id plus the shared name. Matching - // on the name alone resolved to the beta record and evicted it, aborting - // an unrelated sandbox's provisioning task. - let beta = pending_sandbox("sbx-beta", "demo", "beta"); - let pending = pending_map(&[&beta]); - - assert_eq!( - resolve_pending_id(&pending, "sbx-alpha", "demo").unwrap(), - None - ); -} - -#[test] -fn resolve_pending_id_falls_back_to_the_name_when_no_id_is_supplied() { - // Direct driver callers may omit the id; a unique name still resolves. - let alpha = pending_sandbox("sbx-alpha", "demo", "alpha"); - let pending = pending_map(&[&alpha]); - - assert_eq!( - resolve_pending_id(&pending, "", "demo").unwrap().as_deref(), - Some("sbx-alpha") - ); -} - -#[test] -fn resolve_pending_id_rejects_an_ambiguous_name_only_lookup() { - // Two pending sandboxes share a name across workspaces and the driver - // request carries no workspace. Picking either one would make the outcome - // depend on `HashMap` iteration order, so refuse instead. - let alpha = pending_sandbox("sbx-alpha", "demo", "alpha"); - let beta = pending_sandbox("sbx-beta", "demo", "beta"); - let pending = pending_map(&[&alpha, &beta]); - - let err = resolve_pending_id(&pending, "", "demo") - .expect_err("an ambiguous name-only lookup must be rejected"); - assert_eq!(err.code(), tonic::Code::FailedPrecondition); -} - -#[test] -fn resolve_pending_id_returns_none_without_any_identifier() { - // `require_sandbox_identifier` rejects this upstream, but the resolver - // stays total so an empty request can never match an arbitrary record. - let alpha = pending_sandbox("sbx-alpha", "demo", "alpha"); - let pending = pending_map(&[&alpha]); - - assert_eq!(resolve_pending_id(&pending, "", "").unwrap(), None); -} - -#[tokio::test] -async fn remove_pending_sandbox_by_id_keeps_a_same_named_sandbox_in_another_workspace() { - let alpha = pending_sandbox("sbx-alpha", "demo", "alpha"); - let beta = pending_sandbox("sbx-beta", "demo", "beta"); - let driver = driver_with_pending(&[&alpha, &beta]).await; - - let removed = driver - .remove_pending_sandbox("sbx-alpha", "demo") - .await - .expect("an id-scoped removal must succeed") - .expect("the alpha record must be removed"); - - assert_eq!(removed.sandbox.id, "sbx-alpha"); - assert_eq!( - pending_ids(&driver.pending_snapshot_map().await), - ["sbx-beta"] - ); -} - -#[tokio::test] -async fn remove_pending_sandbox_by_a_unique_name_still_removes_the_record() { - let alpha = pending_sandbox("sbx-alpha", "demo", "alpha"); - let driver = driver_with_pending(&[&alpha]).await; - - let removed = driver - .remove_pending_sandbox("", "demo") - .await - .expect("a unique name-only removal must succeed") - .expect("the alpha record must be removed"); - - assert_eq!(removed.sandbox.id, "sbx-alpha"); - assert!(driver.pending_snapshot_map().await.is_empty()); -} - -#[tokio::test] -async fn remove_pending_sandbox_rejects_an_ambiguous_name_and_keeps_both_records() { - let alpha = pending_sandbox("sbx-alpha", "demo", "alpha"); - let beta = pending_sandbox("sbx-beta", "demo", "beta"); - let driver = driver_with_pending(&[&alpha, &beta]).await; - - let err = driver - .remove_pending_sandbox("", "demo") - .await - .map(|record| record.map(|record| record.sandbox.id)) - .expect_err("an ambiguous name-only removal must be rejected"); - - assert_eq!(err.code(), tonic::Code::FailedPrecondition); - assert_eq!( - pending_ids(&driver.pending_snapshot_map().await), - ["sbx-alpha", "sbx-beta"] - ); -} - -#[tokio::test] -async fn pending_snapshot_by_id_ignores_a_same_named_sandbox_in_another_workspace() { - // `GetSandbox` falls through to the pending map when no container exists. - // Resolving by name there leaked another workspace's snapshot. - let beta = pending_sandbox("sbx-beta", "demo", "beta"); - let driver = driver_with_pending(&[&beta]).await; - - assert!( - driver - .pending_snapshot("sbx-alpha", "demo") - .await - .expect("an id-scoped snapshot lookup must succeed") - .is_none() - ); -} - -#[tokio::test] -async fn pending_snapshot_rejects_an_ambiguous_name_only_lookup() { - let alpha = pending_sandbox("sbx-alpha", "demo", "alpha"); - let beta = pending_sandbox("sbx-beta", "demo", "beta"); - let driver = driver_with_pending(&[&alpha, &beta]).await; - - let err = driver - .pending_snapshot("", "demo") - .await - .expect_err("an ambiguous name-only snapshot lookup must be rejected"); - - assert_eq!(err.code(), tonic::Code::FailedPrecondition); -} - -#[tokio::test] -async fn reserve_pending_sandbox_allows_the_same_name_in_a_different_workspace() { - // Sandbox names are unique per workspace, so this is a legitimate create. - let alpha = pending_sandbox("sbx-alpha", "demo", "alpha"); - let beta = pending_sandbox("sbx-beta", "demo", "beta"); - let driver = driver_with_pending(&[&alpha]).await; - - driver - .reserve_pending_sandbox(&beta) - .await - .expect("a same-named sandbox in another workspace must be allowed"); - - assert_eq!( - pending_ids(&driver.pending_snapshot_map().await), - ["sbx-alpha", "sbx-beta"] - ); -} - -#[tokio::test] -async fn reserve_pending_sandbox_rejects_a_duplicate_name_in_the_same_workspace() { - let alpha = pending_sandbox("sbx-alpha", "demo", "alpha"); - let duplicate = pending_sandbox("sbx-other", "demo", "alpha"); - let driver = driver_with_pending(&[&alpha]).await; - - let err = driver - .reserve_pending_sandbox(&duplicate) - .await - .expect_err("a duplicate name within one workspace must be rejected"); - - assert_eq!(err.code(), tonic::Code::AlreadyExists); - assert_eq!( - pending_ids(&driver.pending_snapshot_map().await), - ["sbx-alpha"] - ); -} - -#[tokio::test] -async fn reserve_pending_sandbox_rejects_a_duplicate_id() { - let alpha = pending_sandbox("sbx-alpha", "demo", "alpha"); - let duplicate = pending_sandbox("sbx-alpha", "other-name", "beta"); - let driver = driver_with_pending(&[&alpha]).await; - - let err = driver - .reserve_pending_sandbox(&duplicate) - .await - .expect_err("a duplicate sandbox id must be rejected regardless of workspace"); - - assert_eq!(err.code(), tonic::Code::AlreadyExists); - assert_eq!( - pending_ids(&driver.pending_snapshot_map().await), - ["sbx-alpha"] - ); -} - -fn managed_container_labels( - namespace: &str, - sandbox_id: &str, - sandbox_name: &str, -) -> HashMap { - HashMap::from([ - ( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - ), - (LABEL_SANDBOX_NAMESPACE.to_string(), namespace.to_string()), - (LABEL_SANDBOX_ID.to_string(), sandbox_id.to_string()), - (LABEL_SANDBOX_NAME.to_string(), sandbox_name.to_string()), - ]) -} - -#[test] -fn managed_container_identity_matches_on_id_despite_a_stale_name() { - // Requiring the name to agree with an authoritative id dropped the match - // and made the driver report a live sandbox as absent, stranding the - // container and leaking its token file. - let labels = managed_container_labels("default", "sbx-alpha", "demo"); - - assert!(managed_container_identity_matches( - &labels, - "default", - "sbx-alpha", - "stale-name" - )); -} - -#[test] -fn managed_container_identity_rejects_a_name_match_when_the_id_differs() { - // The mirror of the pending-map fix: a shared name must not stand in for - // an id that explicitly disagrees. - let labels = managed_container_labels("default", "sbx-beta", "demo"); - - assert!(!managed_container_identity_matches( - &labels, - "default", - "sbx-alpha", - "demo" - )); -} - -#[test] -fn managed_container_identity_falls_back_to_the_name_without_an_id() { - let labels = managed_container_labels("default", "sbx-alpha", "demo"); - - assert!(managed_container_identity_matches( - &labels, "default", "", "demo" - )); - assert!(!managed_container_identity_matches( - &labels, "default", "", "other" - )); -} - -#[test] -fn managed_container_identity_matches_nothing_without_an_identifier() { - // The label filters degenerate to "every managed container in the - // namespace" when neither identifier is supplied, so the predicate must - // not wave the container through. - let labels = managed_container_labels("default", "sbx-alpha", "demo"); - - assert!(!managed_container_identity_matches( - &labels, "default", "", "" - )); -} - #[test] -fn managed_container_identity_requires_the_configured_namespace() { - let labels = managed_container_labels("other-namespace", "sbx-alpha", "demo"); - - assert!(!managed_container_identity_matches( - &labels, - "default", - "sbx-alpha", - "demo" - )); -} - -#[tokio::test] -async fn delete_sandbox_reclaims_token_file_when_container_and_pending_are_gone() { - let state_dir = tempfile::tempdir().unwrap(); - let (endpoint, server) = fake_docker_with_no_containers().await; - - temp_env::async_with_vars([("XDG_STATE_HOME", Some(state_dir.path()))], async { - let config = runtime_config(); - let mut driver = test_driver_with_config(config.clone()); - driver.docker = Arc::new( - Docker::connect_with_http(&endpoint, 5, bollard::API_DEFAULT_VERSION).unwrap(), - ); - - // Arrange the leak: token on disk, container gone, `pending` empty. - let token = openshell_core::driver_utils::sandbox_token_path( - "docker-sandbox-tokens", - Some(&config.sandbox_label), - "sandbox-1", - ) - .unwrap(); - - fs::create_dir_all(token.parent().unwrap()).unwrap(); - fs::write(&token, "jwt\n").unwrap(); - - let deleted = driver.delete_sandbox_inner("sandbox-1", "").await.unwrap(); - assert!(!deleted, "nothing was removed, must not claim a deletion"); - assert!(!token.exists(), "token file must be reclaimed"); - }) - .await; - - server.abort(); -} - -#[tokio::test] -async fn delete_sandbox_by_name_only_leaves_the_namespace_directory_alone() { - // `DeleteSandbox` accepts a name without an id. With no id there is no - // token path to derive, so the cleanup must be a no-op: deriving a path - // from an empty id yields `/sandbox.jwt`, whose parent is the - // shared namespace directory. - let state_dir = tempfile::tempdir().unwrap(); - let (endpoint, server) = fake_docker_with_no_containers().await; - - temp_env::async_with_vars([("XDG_STATE_HOME", Some(state_dir.path()))], async { - let config = runtime_config(); - let mut driver = test_driver_with_config(config.clone()); - driver.docker = Arc::new( - Docker::connect_with_http(&endpoint, 5, bollard::API_DEFAULT_VERSION).unwrap(), - ); - - let namespace_dir = openshell_core::driver_utils::sandbox_token_path( - "docker-sandbox-tokens", - Some(&config.sandbox_label), - "sandbox-1", - ) - .unwrap() - .parent() - .and_then(Path::parent) - .unwrap() - .to_path_buf(); - fs::create_dir_all(&namespace_dir).unwrap(); - - let deleted = driver.delete_sandbox_inner("", "sandbox-1").await.unwrap(); - - assert!(!deleted, "nothing was removed, must not claim a deletion"); - assert!( - namespace_dir.is_dir(), - "namespace directory must survive a name-only delete: {}", - namespace_dir.display() - ); - }) - .await; +fn concurrent_container_removal_is_idempotent() { + let removing = BollardError::DockerResponseServerError { + status_code: 409, + message: "removal of container abc123 is already in progress".to_string(), + }; + let other_conflict = BollardError::DockerResponseServerError { + status_code: 409, + message: "container abc123 is running".to_string(), + }; - server.abort(); + assert!(is_removal_in_progress_error(&removing)); + assert!(!is_removal_in_progress_error(&other_conflict)); } diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index e407d1f351..608bb36b52 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -711,14 +711,11 @@ sandbox_label = "docker-dev" # Optional override. When omitted, the gateway derives # https://host.openshell.internal: for this topology. grpc_endpoint = "https://host.openshell.internal:17670" -# Skip the image-pull-and-extract step by pointing at a locally built binary. -supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" -# When supervisor_bin is omitted, Docker extracts /openshell-sandbox from this image. -# Defaults to the gateway version; override to pin a specific build. +# Contains both /openshell-sandbox and /openshell-supervisor. Defaults to the +# gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" network_name = "openshell-docker" host_gateway_ip = "172.17.0.1" -ssh_socket_path = "/run/openshell/ssh.sock" # Unsafe operator override. Host bind mounts, including Docker local-driver # bind-backed volumes, expose gateway-host paths inside sandboxes and can # negate OpenShell isolation and filesystem controls. diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 30363b50c6..00282070d0 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -40,40 +40,13 @@ an exited canonical process remains a terminal sandbox result. Exit code zero produces `Completed`; a nonzero or signal-normalized exit produces `Error` with the exact exit code. Driver and supervisor failures remain `Error`. -## Build with Selected Compute Drivers - -Source builds of `openshell-gateway` can include any subset of the Docker, -Podman, Kubernetes, VM, and MXC drivers. Enable the corresponding -`compute-driver-docker`, `compute-driver-podman`, `compute-driver-kubernetes`, -`compute-driver-vm`, or `compute-driver-mxc` Cargo features. For example, build -a Docker-only gateway with telemetry support: - -```shell -cargo build --release -p openshell-gateway --no-default-features --features telemetry,compute-driver-docker -``` - -On Windows, select only MXC with: - -```shell -cargo build --release -p openshell-gateway --no-default-features --features telemetry,compute-driver-mxc,bundled-z3 -``` - -The default `in-tree-compute-drivers` feature retains the full platform driver -set, including MXC on Windows. MXC links only on Windows. The other four -features link drivers on non-Windows platforms; on Windows they install -registrations that report the driver as unsupported. A build with no default -features and no driver features connects to external drivers only. -Auto-detection probes only compiled registrations. -To select a driver omitted from a custom build, configure its external -`socket_path` as described below. - ## Configure a Compute Driver -Configure the compute driver on the gateway. Current releases accept one driver per gateway. Set the singular `compute_driver` key in the gateway TOML file: +Configure the compute driver on the gateway. Current releases accept one driver per gateway. Set `compute_drivers` in the gateway TOML file: ```toml [openshell.gateway] -compute_driver = "docker" +compute_drivers = ["docker"] ``` Reserved built-in values are `docker`, `podman`, `kubernetes`, `vm`, and `mxc`. @@ -81,17 +54,15 @@ The `mxc` driver is available only in native Windows gateway builds. Non-reserved names select an extension driver and require a `socket_path` in `[openshell.drivers.]`. -When `compute_driver` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Docker must respond on a known API socket. Podman first probes known API sockets and then asks the `podman` CLI for the active native or machine-backed socket. The VM driver is never auto-detected; configure it explicitly with `compute_driver = "vm"` or set `OPENSHELL_COMPUTE_DRIVER=vm` in the launch environment. - -`compute_driver` accepts exactly one scalar driver name. The legacy `compute_drivers` list is rejected by schema version 2. +When `compute_drivers` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Docker must respond on a known API socket. Podman first probes known API sockets and then asks the `podman` CLI for the active native or machine-backed socket. The VM driver is never auto-detected; configure it explicitly with `compute_drivers = ["vm"]` or set `OPENSHELL_DRIVERS=vm` in the launch environment. Common gateway options: | Gateway TOML option | Description | |---|---| -| `compute_driver = ""` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | +| `compute_drivers = [""]` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | -Set driver-specific values such as sandbox images, callback endpoints, network names, and VM sizing in the gateway TOML file. A TLS-enabled gateway-managed Docker, Podman, or VM driver requires a complete `guest_tls_ca`, `guest_tls_cert`, and `guest_tls_key` bundle in `[openshell.gateway]`; package-managed local TLS supplies it automatically. Driver tables reject those gateway-owned fields. Kubernetes projects guest TLS through a Secret instead. See the [Gateway Configuration File](./gateway-config) reference for the full schema and migration steps. +Set driver-specific values such as sandbox images, callback endpoints, network names, TLS material, and VM sizing in the gateway TOML file. See the [Gateway Configuration File](./gateway-config) reference for the full `[openshell.drivers.]` schema. Extension drivers use the same `compute_driver.proto` gRPC surface as the managed VM driver. For an out-of-tree driver, choose a driver name and point @@ -99,7 +70,7 @@ the gateway at the Unix socket the operator has already provisioned: ```toml [openshell.gateway] -compute_driver = "kyma" +compute_drivers = ["kyma"] [openshell.drivers.kyma] socket_path = "/run/openshell/kyma.sock" @@ -110,8 +81,8 @@ socket path. The endpoint replaces normal driver construction for that name, including canonical built-in names: ```shell -openshell-gateway --compute-driver kyma --compute-driver-socket /run/openshell/kyma.sock -openshell-gateway --compute-driver docker --compute-driver-socket /run/openshell/docker.sock +openshell-gateway --drivers kyma --compute-driver-socket /run/openshell/kyma.sock +openshell-gateway --drivers docker --compute-driver-socket /run/openshell/docker.sock ``` The gateway connects to the operator-provided endpoint; it does not provision @@ -186,7 +157,7 @@ the gateway. If the primary listener covers that address, the gateway reuses it and sandbox JWT authentication restricts the supervisor to its callback RPC allowlist. If the primary listener is not reachable through that address, the gateway creates an additional callback-only listener. Use the primary endpoint -for CLI, administrator, health, reflection, provider management, and +for CLI, administrator, health, reflection, inference-route management, and HTTP requests. A `PermissionDenied` response from an additional callback-only listener is expected for those requests. Do not broaden the primary listener to `0.0.0.0` solely to make sandbox callbacks reachable. @@ -195,7 +166,7 @@ to `0.0.0.0` solely to make sandbox callbacks reachable. [Docker](https://www.docker.com/get-started/)-backed sandboxes run as containers on the gateway host. Use Docker for local development, single-machine gateways, and hosts that already use Docker Desktop or Docker Engine. -The gateway talks to the Docker daemon to create sandbox containers. +The gateway talks to the Docker daemon to create sandbox containers. Docker is also required for local image builds from directories or Dockerfiles. Docker Desktop and compatible macOS runtimes route `host.openshell.internal` through an IPv4 host-gateway alias. The gateway reuses an IPv4 primary listener @@ -204,7 +175,7 @@ that already covers loopback. Otherwise, the Docker driver requests a separate For maintainer-level implementation details, refer to the [Docker driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-docker/README.md). -Select Docker with `compute_driver = "docker"` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_label`, `supervisor_bin`, `supervisor_image`, `image_pull_policy`, `ssh_socket_path`, and `sandbox_pids_limit` in `[openshell.drivers.docker]`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. +Select Docker with `compute_drivers = ["docker"]` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `supervisor_image`, `image_pull_policy`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. The supervisor image must contain both `/openshell-sandbox` and `/openshell-supervisor`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. When operating `openshell-driver-docker` as an external driver, set `OPENSHELL_OTLP_ENDPOINT` to export its spans. The driver continues W3C trace @@ -282,27 +253,7 @@ The gateway talks to the Podman API socket. The Podman driver requires Podman 5. For maintainer-level implementation details, refer to the [Podman driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/README.md) and [Podman networking notes](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/NETWORKING.md). -Select Podman with `compute_driver = "podman"` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `ssh_socket_path`, and `sandbox_pids_limit` in `[openshell.drivers.podman]`. - -### macOS Podman Socket Path - -On macOS, Homebrew-installed Podman does not create the default socket path -that the driver probes (`~/.local/share/containers/podman/machine/podman.sock`). -The actual API socket lives under `/var/folders/` in a path that macOS can -rotate after a reboot. - -If the gateway fails with `Podman socket not found; is podman machine running?` -while `podman machine list` shows a running machine, set the -`OPENSHELL_PODMAN_SOCKET` environment variable to the dynamic socket path: - -```shell -export OPENSHELL_PODMAN_SOCKET="$(podman machine inspect --format '{{.ConnectionInfo.PodmanSocket.Path}}')" -``` - -Add this to your shell profile or gateway launch environment so it resolves -correctly after each reboot. Alternatively, set `socket_path` in -`[openshell.drivers.podman]` to the current path, but note that the path may -change when macOS rotates `/var/folders/`. +Select Podman with `compute_drivers = ["podman"]` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `sandbox_ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. Podman sandboxes default to a 45-second graceful stop window before Podman escalates from `SIGTERM` to `SIGKILL`. Set `stop_timeout_secs` in gateway config, or `OPENSHELL_STOP_TIMEOUT` for the standalone driver, when a local runtime needs a different teardown window. @@ -316,12 +267,6 @@ stopped sandboxes alone. For proxy-required networks, the Podman driver also accepts the corporate egress proxy keys `https_proxy`, `no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, and `proxy_connect_by_hostname`. The supervisor chains policy-approved TLS tunnels through the proxy with HTTP CONNECT instead of dialing destinations directly. See the [Gateway Configuration File](./gateway-config) reference for the full contract, including the cleartext-credential acknowledgement and the validated-IP CONNECT behavior. -Podman preserves its runtime-selected AppArmor profile when -`app_armor_profile` is omitted. Set `Unconfined` explicitly only when the -supervisor's mount setup requires it. Explicit `RuntimeDefault` and -`Localhost/` selections fail startup when Podman reports that AppArmor -is unavailable. - On macOS with `podman machine`, the driver uses gvproxy's host-loopback IP, `192.168.127.254`, for sandbox host aliases by default. Set `host_gateway_ip` only when your Podman machine uses a non-standard host-loopback address. On Linux, an empty `host_gateway_ip` keeps Podman's `host-gateway` resolver behavior. Direct local callbacks from rootless Podman require Podman to report the pasta network helper. Slirp4netns, other helpers, and Podman versions that do not report their helper require an explicitly remote `grpc_endpoint`; otherwise the gateway fails startup rather than leaving sandbox callbacks unreachable. Rootful Podman continues to use the configured network's bridge gateway address. ### Podman Driver Config Mounts @@ -402,25 +347,19 @@ For maintainer-level implementation details, refer to the [VM driver README](htt The VM driver is opt-in. Release packages can install `openshell-driver-vm`, but the gateway does not select it unless you configure the driver explicitly. -Enable VM by setting `compute_driver = "vm"` in the gateway TOML file: +Enable VM by setting `compute_drivers = ["vm"]` in the gateway TOML file: ```toml [openshell.gateway] -compute_driver = "vm" +compute_drivers = ["vm"] ``` -For a launch-time override, set `OPENSHELL_COMPUTE_DRIVER=vm` in the gateway environment and restart the service. +For a launch-time override, set `OPENSHELL_DRIVERS=vm` in the gateway environment and restart the service. -Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, and `krun_log_level` in `[openshell.drivers.vm]`. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. The VM socket path is managed by the gateway and is not configurable through remote endpoint settings. +Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, `krun_log_level`, and `guest_tls_*` in `[openshell.drivers.vm]`. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. The VM socket path is managed by the gateway and is not configurable through remote endpoint settings. The gateway starts `openshell-driver-vm` over a private Unix socket and passes its process ID so the driver can reject unexpected local clients. The driver's standalone TCP listener is disabled unless `--allow-unauthenticated-tcp` is set for local development. -Scripts that invoke the experimental standalone driver directly must use -`--grpc-endpoint` and the `--upstream-proxy*` option family. Schema v2 removes -the previous `--openshell-endpoint`, `--https-proxy`, `--no-proxy`, and -`--proxy-*` spellings; gateway-managed deployments do not use those options -directly. - ### Local image resolution The VM driver resolves sandbox images from a local container engine before falling back to registry pulls. It tries Docker first, then uses the same Podman socket discovery as the Podman driver. On Linux with Podman, enable the API socket so the driver can find local images: @@ -467,24 +406,23 @@ owner references or use the sandbox ServiceAccount. The operator namespace allowlist is a trust grant, not a tenant isolation mechanism. -Helm deployments set Kubernetes driver values through the chart. Canonical TOML places `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`; schema version 2 rejects their historical `[openshell.gateway]` locations. +Helm deployments set Kubernetes driver values through the chart. For maintainer-level implementation details, refer to the [Kubernetes driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-kubernetes/README.md). | Gateway configuration | Helm value | Description | |---|---|---| -| `compute_driver = "kubernetes"` | Not applicable | Select the Kubernetes compute driver. | +| `compute_drivers = ["kubernetes"]` | Not applicable | Select the Kubernetes compute driver. | | `[openshell.drivers.kubernetes].namespace` | `server.sandboxNamespace` | Set the namespace for sandbox resources. The Helm chart defaults to the release namespace when left empty. | -| `[openshell.drivers.kubernetes].service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the Kubernetes driver's TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | -| `[openshell.drivers.kubernetes].enable_user_namespaces` | `server.enableUserNamespaces` | Enable Kubernetes user namespaces for sandbox pods. | +| `service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the Kubernetes driver's TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | | `default_image` | `server.sandboxImage` | Set the default sandbox image. | -| `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the canonical sandbox pull policy: `always`, `if_not_present`, or `never`. `newer` is Podman-only. | +| `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the Kubernetes image pull policy for sandbox pods. | | `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image-pull Secrets to sandbox pods. Managed mode copies these explicitly named Secrets from the configured source namespace into each workspace namespace. In shared and operator modes, the Secrets must already exist in the sandbox namespace. | | `[managed_ssh_ingress]` | `networkPolicy.enabled` | In managed mode, create an SSH ingress policy in every workspace namespace. Helm configures the gateway namespace and pod selector automatically. Operator mode leaves namespace policy management to the platform operator. | -| `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. Raw TOML and the standalone Kubernetes driver require an explicit endpoint because the sandbox namespace does not identify the gateway Service. Helm derives it from the release's gateway Service when the value is empty. | +| `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. | | `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. | | `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the supervisor image that provides the `openshell-sandbox` binary. The default repository with an empty tag uses the version-pinned image built into the gateway. Changing the repository uses the effective gateway image tag, while setting a tag pins that version explicitly. | -| `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the canonical supervisor pull policy: `always`, `if_not_present`, or `never`. `newer` is Podman-only. | +| `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | | `supervisor_sideload_method` | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect from cluster version. Set to `image-volume` to mount the supervisor OCI image directly as a volume (requires Kubernetes 1.33+ with the ImageVolume feature gate; GA in 1.36), or `init-container` to copy it through an init container on older clusters. | | `topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, or `sidecar` to move pod-level network enforcement and the gateway session into a dedicated sidecar. | | `https_proxy` | `upstreamProxy.url` | Set the operator-owned `http://host:port` corporate forward proxy used for policy-approved TLS CONNECT egress. | @@ -675,7 +613,7 @@ The resolved UID/GID appear in: ### VM Driver -The VM driver preserves an image-provided `sandbox` account when `sandbox_uid` and `sandbox_gid` are omitted. Images without that account use UID/GID `1000`. Explicit values in `[openshell.drivers.vm]` override the image account. Persisted overlays retain the UID/GID recorded when they were created. An unmarked overlay recovers identity from concrete overlay or prepared-image state, an explicit override, or the current image; the driver never assigns legacy `10001:10001` without persisted evidence. +The VM driver injects the sandbox UID into the rootfs guest's `/etc/passwd`, `/etc/group`, and `/etc/gshadow` during rootfs preparation. Default UID is `10001`; configure `sandbox_uid` in `[openshell.drivers.vm]` to use a different value. ### Custom Images diff --git a/e2e/python/test_sandbox_policy.py b/e2e/python/test_sandbox_policy.py index d2ce47e2e0..04c516001a 100644 --- a/e2e/python/test_sandbox_policy.py +++ b/e2e/python/test_sandbox_policy.py @@ -1,9 +1,16 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +"""Python SDK policy integration tests. + +Transparent network interception is exercised by the Rust E2E suites, which +cover TCP, DNS, L7, SSRF, policy reload, and credential rewriting without a +workload-visible forward-proxy endpoint. This module keeps the SDK-level +policy checks that do not depend on the retired ``10.200.0.1:3128`` contract. +""" + from __future__ import annotations -import json from typing import TYPE_CHECKING import grpc @@ -14,34 +21,23 @@ if TYPE_CHECKING: from collections.abc import Callable - from openshell import Sandbox, SandboxClient + from openshell import Sandbox -# ============================================================================= -# Policy helpers -# ============================================================================= - _BASE_FILESYSTEM = sandbox_pb2.FilesystemPolicy( include_workdir=True, read_only=["/usr", "/lib", "/etc", "/app", "/var/log", "/proc", "/dev/urandom"], read_write=["/sandbox", "/tmp"], ) _BASE_LANDLOCK = sandbox_pb2.LandlockPolicy(compatibility="best_effort") -_BASE_PROCESS = sandbox_pb2.ProcessPolicy(run_as_user="sandbox", run_as_group="sandbox") -# Standard proxy address inside the sandbox network namespace -_PROXY_HOST = "10.200.0.1" -_PROXY_PORT = 3128 -# example.com keeps the wildcard test on public DNS while avoiding sslip.io -# rewrites that can resolve to internal ranges in CI. -_PUBLIC_WILDCARD_SUFFIX = "example.com" -_PUBLIC_WILDCARD_PATTERN = f"*.{_PUBLIC_WILDCARD_SUFFIX}" -_PUBLIC_WILDCARD_SUBDOMAIN = f"www.{_PUBLIC_WILDCARD_SUFFIX}" +_BASE_PROCESS = sandbox_pb2.ProcessPolicy( + run_as_user="sandbox", run_as_group="sandbox" +) def _base_policy( network_policies: dict[str, sandbox_pb2.NetworkPolicyRule] | None = None, ) -> sandbox_pb2.SandboxPolicy: - """Build a sandbox policy with standard filesystem/process/landlock settings.""" return sandbox_pb2.SandboxPolicy( version=1, filesystem=_BASE_FILESYSTEM, @@ -51,318 +47,6 @@ def _base_policy( ) -def _policy_for_python_proxy_tests() -> sandbox_pb2.SandboxPolicy: - return _base_policy( - network_policies={ - "python": sandbox_pb2.NetworkPolicyRule( - name="python", - endpoints=[ - sandbox_pb2.NetworkEndpoint(host="api.openai.com", port=443) - ], - binaries=[ - sandbox_pb2.NetworkBinary(path="/sandbox/.uv/python/**/python*") - ], - ) - }, - ) - - -# ============================================================================= -# Shared test function factories -# -# cloudpickle serializes module-level functions by reference (module + name). -# The sandbox doesn't have this module, so deserialization fails. These -# factories return closures that cloudpickle serializes by value instead. -# ============================================================================= - - -def _proxy_connect(): - """Return a closure that sends a raw CONNECT and returns the status line.""" - - def fn(host, port): - import socket - - conn = socket.create_connection(("10.200.0.1", 3128), timeout=10) - try: - conn.sendall( - f"CONNECT {host}:{port} HTTP/1.1\r\nHost: {host}\r\n\r\n".encode() - ) - return conn.recv(256).decode("latin1") - finally: - conn.close() - - return fn - - -def _proxy_connect_then_http(): - """Return a closure that CONNECTs, does TLS + HTTP, returns JSON string.""" - - def fn(host, port, method="GET", path="/"): - import json as _json - import socket - import ssl - - conn = socket.create_connection(("10.200.0.1", 3128), timeout=30) - try: - conn.sendall( - f"CONNECT {host}:{port} HTTP/1.1\r\nHost: {host}\r\n\r\n".encode() - ) - connect_resp = conn.recv(256).decode("latin1") - if "200" not in connect_resp: - return _json.dumps( - {"connect_status": connect_resp.strip(), "http_status": 0} - ) - - sock = conn - if port == 443: - import os - - ctx = ssl.create_default_context() - ca_file = os.environ.get("SSL_CERT_FILE") - if ca_file: - ctx.load_verify_locations(ca_file) - sock = ctx.wrap_socket(conn, server_hostname=host) - - sock.settimeout(15) - - request = ( - f"{method} {path} HTTP/1.1\r\nHost: {host}\r\nConnection: close\r\n\r\n" - ) - sock.sendall(request.encode()) - - # Read response. The L7 relay loops back to parse the next - # request after relaying, so neither side closes — read until - # we have headers, then drain body with a short timeout. - data = b"" - while b"\r\n\r\n" not in data: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - - # Drain body with short timeout - sock.settimeout(2) - while len(data) < 65536: - try: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - except (socket.timeout, TimeoutError): - break - - response = data.decode("latin1", errors="replace") - status_line = response.split("\r\n")[0] if response else "" - status_code = ( - int(status_line.split()[1]) if len(status_line.split()) >= 2 else 0 - ) - - header_end = response.find("\r\n\r\n") - headers_raw = response[:header_end] if header_end > 0 else "" - body = response[header_end + 4 :] if header_end > 0 else "" - - return _json.dumps( - { - "connect_status": connect_resp.strip(), - "http_status": status_code, - "headers": headers_raw, - "body": body, - } - ) - finally: - conn.close() - - return fn - - -def _read_openshell_log(): - """Return a closure that reads the openshell log file(s). - - Since the sandbox uses a rolling file appender, logs are written to - date-stamped files like ``/var/log/openshell.YYYY-MM-DD.log`` instead - of a single ``/var/log/openshell.log``. This helper globs for all - matching files so tests work with both the legacy and rolling layouts. - """ - - def fn(): - import glob - - logs = [] - for path in sorted(glob.glob("/var/log/openshell*.log*")): - try: - with open(path) as f: - logs.append(f.read()) - except (FileNotFoundError, PermissionError): - pass - return "\n".join(logs) - - return fn - - -def _forward_proxy_with_server(): - """Return a closure that starts an HTTP server and sends a forward proxy request. - - The closure starts a minimal HTTP server on the given port inside the sandbox, - then sends a plain HTTP forward proxy request (non-CONNECT) through the sandbox - proxy and returns the raw response. - """ - - def fn(proxy_host, proxy_port, target_host, target_port): - import socket - import threading - import time - from http.server import BaseHTTPRequestHandler, HTTPServer - - class Handler(BaseHTTPRequestHandler): - def do_GET(self): - self.send_response(200) - body = b"forward-proxy-ok" - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, *args): - pass # suppress log output - - srv = HTTPServer(("0.0.0.0", int(target_port)), Handler) - threading.Thread(target=srv.handle_request, daemon=True).start() - time.sleep(0.5) - - conn = socket.create_connection((proxy_host, int(proxy_port)), timeout=10) - try: - req = ( - f"GET http://{target_host}:{target_port}/test HTTP/1.1\r\n" - f"Host: {target_host}:{target_port}\r\n\r\n" - ) - conn.sendall(req.encode()) - data = b"" - conn.settimeout(5) - try: - while True: - chunk = conn.recv(4096) - if not chunk: - break - data += chunk - except socket.timeout: - pass - return data.decode("latin1") - finally: - conn.close() - srv.server_close() - - return fn - - -def _forward_proxy_raw(): - """Return a closure that sends a forward proxy request (no server needed). - - For testing deny cases — sends the request and returns whatever the proxy - responds with. - """ - - def fn(proxy_host, proxy_port, target_url): - import socket - from urllib.parse import urlparse - - conn = socket.create_connection((proxy_host, int(proxy_port)), timeout=10) - try: - parsed = urlparse(target_url) - host_header = parsed.netloc or parsed.hostname - req = f"GET {target_url} HTTP/1.1\r\nHost: {host_header}\r\n\r\n" - conn.sendall(req.encode()) - return conn.recv(4096).decode("latin1") - finally: - conn.close() - - return fn - - -def _proxy_connect_then_http_with_server(): - """Return a closure that starts a local HTTP server and sends CONNECT+HTTP.""" - - def fn(proxy_host, proxy_port, target_host, target_port, method="GET", path="/"): - import json as _json - import socket - import threading - import time - from http.server import BaseHTTPRequestHandler, HTTPServer - - class Handler(BaseHTTPRequestHandler): - def do_GET(self): - self.send_response(200) - body = b"connect-server-ok" - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def do_POST(self): - self.send_response(200) - body = b"connect-server-ok" - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, *args): - pass - - srv = HTTPServer(("0.0.0.0", int(target_port)), Handler) - threading.Thread(target=srv.handle_request, daemon=True).start() - time.sleep(0.5) - - conn = socket.create_connection((proxy_host, int(proxy_port)), timeout=10) - try: - conn.sendall( - f"CONNECT {target_host}:{target_port} HTTP/1.1\r\nHost: {target_host}\r\n\r\n".encode() - ) - connect_resp = conn.recv(256).decode("latin1") - if "200" not in connect_resp: - return _json.dumps( - {"connect_status": connect_resp.strip(), "http_status": 0} - ) - - request = ( - f"{method} {path} HTTP/1.1\r\n" - f"Host: {target_host}:{target_port}\r\n" - "Connection: close\r\n\r\n" - ) - conn.sendall(request.encode()) - - data = b"" - conn.settimeout(5) - try: - while True: - chunk = conn.recv(4096) - if not chunk: - break - data += chunk - except socket.timeout: - pass - - response = data.decode("latin1", errors="replace") - status_line = response.split("\r\n")[0] if response else "" - status_code = ( - int(status_line.split()[1]) if len(status_line.split()) >= 2 else 0 - ) - - header_end = response.find("\r\n\r\n") - headers_raw = response[:header_end] if header_end > 0 else "" - body = response[header_end + 4 :] if header_end > 0 else "" - - return _json.dumps( - { - "connect_status": connect_resp.strip(), - "http_status": status_code, - "headers": headers_raw, - "body": body, - } - ) - finally: - conn.close() - srv.server_close() - - return fn - - def test_policy_applies_to_exec_commands( sandbox: Callable[..., Sandbox], ) -> None: @@ -379,8 +63,7 @@ def write_allowed_files() -> str: Path("/tmp/allowed.txt").write_text("ok") return "ok" - spec = datamodel_pb2.SandboxSpec(policy=_policy_for_python_proxy_tests()) - + spec = datamodel_pb2.SandboxSpec(policy=_base_policy()) with sandbox(spec=spec, delete_on_exit=True) as policy_sandbox: user_result = policy_sandbox.exec_python(current_user) assert user_result.exit_code == 0, user_result.stderr @@ -391,1661 +74,40 @@ def write_allowed_files() -> str: assert file_result.stdout.strip() == "ok" -def test_policy_blocks_unauthorized_proxy_connect( - sandbox: Callable[..., Sandbox], -) -> None: - spec = datamodel_pb2.SandboxSpec(policy=_policy_for_python_proxy_tests()) - with sandbox(spec=spec, delete_on_exit=True) as policy_sandbox: - proxy_result = policy_sandbox.exec_python( - _proxy_connect(), args=("example.com", 443) - ) - assert proxy_result.exit_code == 0, proxy_result.stderr - assert "403" in proxy_result.stdout - - -# ============================================================================= -# L4 Tests -- Connection-level OPA policy (host:port + binary identity) -# ============================================================================= -# -# L4-1: No network policies -> all CONNECT requests denied -# L4-2: Wildcard binary (/**) + specific endpoint -> any binary can connect -# but non-listed endpoints still denied -# L4-3: Binary-restricted policy -> matched binary allowed, others denied -# L4-4: Correct endpoint, wrong port -> denied -# L4-5: Multiple disjoint policies -> cross-policy access denied -# L4-6: Non-CONNECT HTTP method -> rejected with 405 -# L4-7: Log fields are structured correctly (action, binary, policy, engine) -# ============================================================================= - - -def test_l4_no_policy_denies_all( - sandbox: Callable[..., Sandbox], -) -> None: - """L4-1: No matching endpoint in any network policy -> CONNECT denied. - - We need at least one network policy so the proxy and network namespace - start (empty network_policies disables networking entirely, including - socket syscalls). The policy allows python->example.com:443 but - api.anthropic.com:443 should still be denied. - """ - policy = _base_policy( - network_policies={ - "other": sandbox_pb2.NetworkPolicyRule( - name="other", - endpoints=[ - sandbox_pb2.NetworkEndpoint(host="example.com", port=443), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python(_proxy_connect(), args=("api.anthropic.com", 443)) - assert result.exit_code == 0, result.stderr - assert "403" in result.stdout - - -def test_l4_wildcard_binary_allows_any_binary( - sandbox: Callable[..., Sandbox], -) -> None: - """L4-2: Wildcard binary glob allows python (and anything else) to connect.""" - policy = _base_policy( - network_policies={ - "wildcard": sandbox_pb2.NetworkPolicyRule( - name="wildcard", - endpoints=[ - sandbox_pb2.NetworkEndpoint(host="api.anthropic.com", port=443), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - # Python can reach the allowed endpoint - result = sb.exec_python(_proxy_connect(), args=("api.anthropic.com", 443)) - assert result.exit_code == 0, result.stderr - assert "200" in result.stdout - - # Non-listed endpoint is still denied - result = sb.exec_python(_proxy_connect(), args=("example.com", 443)) - assert result.exit_code == 0, result.stderr - assert "403" in result.stdout - - -def test_l4_binary_restricted_denies_wrong_binary( - sandbox: Callable[..., Sandbox], -) -> None: - """L4-3: Policy restricted to specific binary denies others. - - Policy allows /usr/bin/curl -> api.anthropic.com:443. - Python (exec_python uses python) should be denied. - """ - policy = _base_policy( - network_policies={ - "curl_only": sandbox_pb2.NetworkPolicyRule( - name="curl_only", - endpoints=[ - sandbox_pb2.NetworkEndpoint(host="api.anthropic.com", port=443), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/usr/bin/curl")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - # Python is NOT the allowed binary -> denied - result = sb.exec_python(_proxy_connect(), args=("api.anthropic.com", 443)) - assert result.exit_code == 0, result.stderr - assert "403" in result.stdout - - -def test_l4_wrong_port_denied( - sandbox: Callable[..., Sandbox], -) -> None: - """L4-4: Correct host but wrong port -> denied.""" - policy = _base_policy( - network_policies={ - "anthropic": sandbox_pb2.NetworkPolicyRule( - name="anthropic", - endpoints=[ - sandbox_pb2.NetworkEndpoint(host="api.anthropic.com", port=443), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - # Port 443 -> allowed - result = sb.exec_python(_proxy_connect(), args=("api.anthropic.com", 443)) - assert result.exit_code == 0, result.stderr - assert "200" in result.stdout - - # Port 80 -> denied - result = sb.exec_python(_proxy_connect(), args=("api.anthropic.com", 80)) - assert result.exit_code == 0, result.stderr - assert "403" in result.stdout - - -def test_l4_cross_policy_denied( - sandbox: Callable[..., Sandbox], -) -> None: - """L4-5: Multiple disjoint policies -> cross-policy access denied. - - Policy A: python -> api.anthropic.com:443 - Policy B: curl -> example.com:443 - Python should NOT reach example.com (that's curl's policy). - """ - policy = _base_policy( - network_policies={ - "anthropic": sandbox_pb2.NetworkPolicyRule( - name="anthropic", - endpoints=[ - sandbox_pb2.NetworkEndpoint(host="api.anthropic.com", port=443), - ], - binaries=[ - sandbox_pb2.NetworkBinary(path="/sandbox/.uv/python/**/python*") - ], - ), - "other": sandbox_pb2.NetworkPolicyRule( - name="other", - endpoints=[ - sandbox_pb2.NetworkEndpoint(host="example.com", port=443), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/usr/bin/curl")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - # Python -> its own policy endpoint: allowed - result = sb.exec_python(_proxy_connect(), args=("api.anthropic.com", 443)) - assert result.exit_code == 0, result.stderr - assert "200" in result.stdout - - # Python -> curl's policy endpoint: denied - result = sb.exec_python(_proxy_connect(), args=("example.com", 443)) - assert result.exit_code == 0, result.stderr - assert "403" in result.stdout - - -def test_l4_non_connect_method_rejected( - sandbox: Callable[..., Sandbox], -) -> None: - """L4-6: Non-CONNECT HTTP method -> rejected with 403.""" - - def send_get_to_proxy() -> str: - import socket - - conn = socket.create_connection(("10.200.0.1", 3128), timeout=10) - try: - conn.sendall( - b"GET http://example.com/ HTTP/1.1\r\nHost: example.com\r\n\r\n" - ) - return conn.recv(256).decode("latin1") - finally: - conn.close() - - policy = _base_policy( - network_policies={ - "any": sandbox_pb2.NetworkPolicyRule( - name="any", - endpoints=[ - sandbox_pb2.NetworkEndpoint(host="example.com", port=443), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python(send_get_to_proxy) - assert result.exit_code == 0, result.stderr - assert "403" in result.stdout - - -def test_l4_log_fields( - sandbox: Callable[..., Sandbox], -) -> None: - """L4-7: CONNECT log contains structured fields for allow and deny.""" - policy = _base_policy( - network_policies={ - "anthropic": sandbox_pb2.NetworkPolicyRule( - name="anthropic", - endpoints=[ - sandbox_pb2.NetworkEndpoint(host="api.anthropic.com", port=443), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - # Generate an allow - sb.exec_python(_proxy_connect(), args=("api.anthropic.com", 443)) - # Generate a deny - sb.exec_python(_proxy_connect(), args=("example.com", 443)) - - log_result = sb.exec_python(_read_openshell_log()) - assert log_result.exit_code == 0, log_result.stderr - log = log_result.stdout - - # Verify OCSF shorthand fields in allow line - assert "ALLOWED" in log, "Expected ALLOWED in OCSF shorthand" - assert "api.anthropic.com" in log, "Expected destination host in log" - assert "engine:opa" in log, "Expected engine:opa in log context" - - # Verify deny line exists - assert "DENIED" in log, "Expected DENIED in OCSF shorthand" - - -# ============================================================================= -# SSRF Tests -- Internal IP rejection (defense-in-depth) -# -# The proxy resolves DNS before connecting and rejects any destination that -# resolves to a loopback, RFC1918 private, or link-local address. These -# tests verify the check works even when OPA policy explicitly allows the -# internal endpoint. -# -# SSRF-1: Loopback (127.0.0.1) blocked despite OPA allow -# SSRF-2: Cloud metadata (169.254.169.254) blocked despite OPA allow -# SSRF-3: Log shows "internal address" block reason -# ============================================================================= - - -def test_ssrf_blocks_loopback_despite_policy_allow( - sandbox: Callable[..., Sandbox], -) -> None: - """SSRF-1: CONNECT to 127.0.0.1 blocked even with explicit OPA allow.""" - policy = _base_policy( - network_policies={ - "internal": sandbox_pb2.NetworkPolicyRule( - name="internal", - endpoints=[ - sandbox_pb2.NetworkEndpoint(host="127.0.0.1", port=80), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python(_proxy_connect(), args=("127.0.0.1", 80)) - assert result.exit_code == 0, result.stderr - assert "403" in result.stdout - - -def test_ssrf_blocks_metadata_endpoint_despite_policy_allow( - sandbox: Callable[..., Sandbox], -) -> None: - """SSRF-2: CONNECT to 169.254.169.254 blocked even with explicit OPA allow.""" - policy = _base_policy( - network_policies={ - "metadata": sandbox_pb2.NetworkPolicyRule( - name="metadata", - endpoints=[ - sandbox_pb2.NetworkEndpoint(host="169.254.169.254", port=80), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python(_proxy_connect(), args=("169.254.169.254", 80)) - assert result.exit_code == 0, result.stderr - assert "403" in result.stdout - - -def test_ssrf_log_shows_blocked_address( - sandbox: Callable[..., Sandbox], -) -> None: - """SSRF-3: Proxy log includes block reason when SSRF check fires. - - Loopback addresses are always-blocked. Since implicit_allowed_ips_for_ip_host - now skips always-blocked hosts, 127.0.0.1 falls through to the default - resolve_and_reject_internal path which blocks it as an internal address. - The shorthand log should include 'ssrf' and a '[reason:' tag for denied events. - """ - policy = _base_policy( - network_policies={ - "internal": sandbox_pb2.NetworkPolicyRule( - name="internal", - endpoints=[ - sandbox_pb2.NetworkEndpoint(host="127.0.0.1", port=80), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - sb.exec_python(_proxy_connect(), args=("127.0.0.1", 80)) - - log_result = sb.exec_python(_read_openshell_log()) - assert log_result.exit_code == 0, log_result.stderr - log = log_result.stdout - # OCSF shorthand uses "engine:ssrf" for SSRF blocks - assert "engine:ssrf" in log.lower() or "ssrf" in log.lower(), ( - f"Expected SSRF block indicator in proxy log, got:\n{log}" - ) - # Shorthand for denied events should include [reason:...] tag - assert "[reason:" in log.lower(), ( - f"Expected [reason:] tag in denied event shorthand, got:\n{log}" - ) - - -# ============================================================================= -# SSRF Tests -- allowed_ips (CIDR-based private IP access) -# -# When an endpoint has `allowed_ips`, the proxy validates resolved IPs against -# the CIDR allowlist instead of blanket-blocking all private IPs. -# Loopback and link-local remain always-blocked regardless. -# -# SSRF-4: Private IP allowed with allowed_ips (mode 2: host + IPs) -# SSRF-5: Private IP allowed with allowed_ips (mode 3: IPs only, no host) -# SSRF-6: Private IP still blocked without allowed_ips (default behavior) -# SSRF-7: Loopback always blocked even with allowed_ips covering 127.0.0.0/8 -# ============================================================================= - - -def test_ssrf_allowed_ips_permits_private_ip( +def test_conflicting_destination_metadata_is_rejected( sandbox: Callable[..., Sandbox], ) -> None: - """SSRF-4: CONNECT to private IP succeeds when allowed_ips covers it. - - Uses 10.200.0.1 (the proxy's own host-side veth IP) as the target. - The connection attempt will fail at the TCP level (nothing listening on - port 19999) but the proxy should return 200 Connection Established - instead of 403, proving the SSRF check passed. - """ + """The gateway rejects ambiguous endpoint pinning before launch.""" + target = "10.200.0.2" + port = 19876 policy = _base_policy( network_policies={ - "internal": sandbox_pb2.NetworkPolicyRule( - name="internal", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host="10.200.0.1", - port=19999, - allowed_ips=["10.200.0.0/24"], - ), - ], + "user_rule": sandbox_pb2.NetworkPolicyRule( + name="user_rule", + endpoints=[sandbox_pb2.NetworkEndpoint(host=target, port=port)], binaries=[sandbox_pb2.NetworkBinary(path="/**")], ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python(_proxy_connect(), args=("10.200.0.1", 19999)) - assert result.exit_code == 0, result.stderr - # Should get 200 (connection established) — not 403. - # The actual TCP connection may fail but the SSRF check passed. - assert "403" not in result.stdout, ( - "Expected SSRF check to pass with allowed_ips, but got 403" - ) - - -def test_ssrf_allowed_ips_hostless_permits_private_ip( - sandbox: Callable[..., Sandbox], -) -> None: - """SSRF-5: CONNECT to private IP succeeds with hostless allowed_ips (mode 3). - - An endpoint with no host but with allowed_ips matches any hostname on the - given port. The resolved IP must be in the allowlist. - """ - policy = _base_policy( - network_policies={ - "private_net": sandbox_pb2.NetworkPolicyRule( - name="private_net", + "approved_rule": sandbox_pb2.NetworkPolicyRule( + name="approved_rule", endpoints=[ sandbox_pb2.NetworkEndpoint( - # No host — matches any hostname on this port - port=19999, + host=target, + port=port, allowed_ips=["10.200.0.0/24"], - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python(_proxy_connect(), args=("10.200.0.1", 19999)) - assert result.exit_code == 0, result.stderr - assert "403" not in result.stdout, ( - "Expected SSRF check to pass with hostless allowed_ips, but got 403" - ) - - -def test_ssrf_private_ip_allowed_with_literal_ip_host( - sandbox: Callable[..., Sandbox], -) -> None: - """SSRF-6: Private IP allowed when policy host is a literal IP address. - - When the policy endpoint host is a literal IP, the user has explicitly - declared intent. The proxy synthesizes an implicit allowed_ips entry, - so the CONNECT succeeds (200) even without explicit allowed_ips. - """ - policy = _base_policy( - network_policies={ - "internal": sandbox_pb2.NetworkPolicyRule( - name="internal", - endpoints=[ - # No allowed_ips — but host is a literal IP, so implicit - sandbox_pb2.NetworkEndpoint(host="10.200.0.1", port=19999), + ) ], binaries=[sandbox_pb2.NetworkBinary(path="/**")], ), - }, + } ) spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python(_proxy_connect(), args=("10.200.0.1", 19999)) - assert result.exit_code == 0, result.stderr - # Should not get 403 — the SSRF check should pass. - # The actual TCP connection may fail (nothing listening on 19999) - # so recv() might return empty, but 403 must not appear. - assert "403" not in result.stdout, ( - "Expected SSRF check to pass for literal IP host, but got 403" - ) - - -def test_ssrf_loopback_blocked_even_with_allowed_ips( - sandbox: Callable[..., Sandbox], -) -> None: - """SSRF-7: Loopback always blocked even when allowed_ips covers 127.0.0.0/8. - - With always-blocked validation, parse_allowed_ips rejects 127.0.0.0/8 at - connection time (returns Err), so the proxy treats this as "invalid - allowed_ips in policy" and returns 403. The end result is the same: - loopback is never reachable. - """ - policy = _base_policy( - network_policies={ - "internal": sandbox_pb2.NetworkPolicyRule( - name="internal", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host="127.0.0.1", - port=80, - allowed_ips=["127.0.0.0/8"], - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python(_proxy_connect(), args=("127.0.0.1", 80)) - assert result.exit_code == 0, result.stderr - assert "403" in result.stdout, ( - "Expected loopback to be blocked even with allowed_ips" - ) - - -# ============================================================================= -# L7 Tests -- TLS termination HTTPS inspection (Phase 2: tls=terminate) -# -# These tests use api.anthropic.com:443 as a real HTTPS endpoint since the -# sandbox already has proxy connectivity. The ephemeral CA is trusted via -# SSL_CERT_FILE injected into the sandbox environment. -# -# L7-T1: TLS terminate + access=full allows HTTPS requests through -# L7-T2: TLS terminate + access=read-only denies HTTPS POST (enforce) -# L7-T3: TLS terminate + enforcement=audit logs but allows HTTPS POST -# L7-T4: TLS terminate with explicit path rules -# L7-T5: CA trust store is injected (SSL_CERT_FILE, NODE_EXTRA_CA_CERTS) -# L7-T6: L7 deny response is valid JSON with expected fields -# L7-T7: L7 request logging includes structured fields -# L7-T8: Port 443 + protocol=rest without tls=terminate warns (L7 not evaluated) -# L7-T9: Query matcher glob/any allows and denies as expected -# L7-T10: Rule without query matcher allows any query params -# ============================================================================= - - -def test_l7_tls_full_access_allows_all( - sandbox: Callable[..., Sandbox], -) -> None: - """L7-T1: TLS terminate + access=full allows HTTPS GET through.""" - policy = _base_policy( - network_policies={ - "anthropic": sandbox_pb2.NetworkPolicyRule( - name="anthropic", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host="api.anthropic.com", - port=443, - protocol="rest", - tls="terminate", - enforcement="enforce", - access="full", - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python( - _proxy_connect_then_http(), - args=("api.anthropic.com", 443, "GET", "/v1/models"), - ) - assert result.exit_code == 0, result.stderr - resp = json.loads(result.stdout) - assert "200" in resp["connect_status"] - # Upstream returns a real response (likely 401 without auth, but not 403 from proxy) - assert resp["http_status"] != 0 - assert resp["http_status"] != 403 # Not a proxy deny - - -def test_l7_tls_read_only_denies_post( - sandbox: Callable[..., Sandbox], -) -> None: - """L7-T2: TLS terminate + access=read-only denies HTTPS POST (enforce).""" - policy = _base_policy( - network_policies={ - "anthropic": sandbox_pb2.NetworkPolicyRule( - name="anthropic", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host="api.anthropic.com", - port=443, - protocol="rest", - tls="terminate", - enforcement="enforce", - access="read-only", - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - # GET should be allowed through (read-only permits GET) - get_result = sb.exec_python( - _proxy_connect_then_http(), - args=("api.anthropic.com", 443, "GET", "/v1/models"), - ) - assert get_result.exit_code == 0, get_result.stderr - get_resp = json.loads(get_result.stdout) - assert get_resp["http_status"] != 403 # Not proxy denied - - # POST should be denied by the proxy with 403 - post_result = sb.exec_python( - _proxy_connect_then_http(), - args=("api.anthropic.com", 443, "POST", "/v1/messages"), - ) - assert post_result.exit_code == 0, post_result.stderr - post_resp = json.loads(post_result.stdout) - assert post_resp["http_status"] == 403 - assert "policy_denied" in post_resp["body"] - - -def test_l7_tls_audit_mode_allows_but_logs( - sandbox: Callable[..., Sandbox], -) -> None: - """L7-T3: TLS terminate + enforcement=audit logs but allows HTTPS POST.""" - policy = _base_policy( - network_policies={ - "anthropic": sandbox_pb2.NetworkPolicyRule( - name="anthropic", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host="api.anthropic.com", - port=443, - protocol="rest", - tls="terminate", - enforcement="audit", - access="read-only", - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - # POST goes through in audit mode (not denied) - post_result = sb.exec_python( - _proxy_connect_then_http(), - args=("api.anthropic.com", 443, "POST", "/v1/messages"), - ) - assert post_result.exit_code == 0, post_result.stderr - post_resp = json.loads(post_result.stdout) - # Should NOT be 403 from proxy -- traffic is forwarded - assert post_resp["http_status"] != 403 - - # Log should contain audit decision - log_result = sb.exec_python(_read_openshell_log()) - assert log_result.exit_code == 0, log_result.stderr - log = log_result.stdout - # OCSF shorthand: audit decisions show as ALLOWED (audit mode allows through) - assert "HTTP:" in log, "Expected OCSF HTTP activity event in log" - assert "ALLOWED" in log, "Expected ALLOWED for audit-mode decision" - - -def test_l7_tls_explicit_path_rules( - sandbox: Callable[..., Sandbox], -) -> None: - """L7-T4: TLS terminate with explicit path rules.""" - policy = _base_policy( - network_policies={ - "anthropic": sandbox_pb2.NetworkPolicyRule( - name="anthropic", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host="api.anthropic.com", - port=443, - protocol="rest", - tls="terminate", - enforcement="enforce", - rules=[ - sandbox_pb2.L7Rule( - allow=sandbox_pb2.L7Allow(method="GET", path="/v1/**"), - ), - ], - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - # GET /v1/models -> allowed (matches /v1/**) - get_result = sb.exec_python( - _proxy_connect_then_http(), - args=("api.anthropic.com", 443, "GET", "/v1/models"), - ) - assert get_result.exit_code == 0, get_result.stderr - get_resp = json.loads(get_result.stdout) - assert get_resp["http_status"] != 403 - - # POST /v1/messages -> denied (no POST rule) - post_result = sb.exec_python( - _proxy_connect_then_http(), - args=("api.anthropic.com", 443, "POST", "/v1/messages"), - ) - assert post_result.exit_code == 0, post_result.stderr - post_resp = json.loads(post_result.stdout) - assert post_resp["http_status"] == 403 - - # GET /v2/anything -> denied (path doesn't match /v1/**) - v2_result = sb.exec_python( - _proxy_connect_then_http(), - args=("api.anthropic.com", 443, "GET", "/v2/anything"), - ) - assert v2_result.exit_code == 0, v2_result.stderr - v2_resp = json.loads(v2_result.stdout) - assert v2_resp["http_status"] == 403 - - -def test_l7_tls_ca_trust_store_injected( - sandbox: Callable[..., Sandbox], -) -> None: - """L7-T5: Sandbox CA is injected into trust store environment variables.""" - - def check_ca_env() -> str: - import json as _json - import os - - return _json.dumps( - { - "SSL_CERT_FILE": os.environ.get("SSL_CERT_FILE", ""), - "NODE_EXTRA_CA_CERTS": os.environ.get("NODE_EXTRA_CA_CERTS", ""), - "REQUESTS_CA_BUNDLE": os.environ.get("REQUESTS_CA_BUNDLE", ""), - "CURL_CA_BUNDLE": os.environ.get("CURL_CA_BUNDLE", ""), - "ca_cert_exists": os.path.exists("/etc/openshell-tls/openshell-ca.pem"), - "bundle_exists": os.path.exists("/etc/openshell-tls/ca-bundle.pem"), - } - ) - - policy = _base_policy( - network_policies={ - "any": sandbox_pb2.NetworkPolicyRule( - name="any", - endpoints=[ - sandbox_pb2.NetworkEndpoint(host="example.com", port=443), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python(check_ca_env) - assert result.exit_code == 0, result.stderr - env = json.loads(result.stdout) - assert env["ca_cert_exists"], "openshell-ca.pem should exist" - assert env["bundle_exists"], "ca-bundle.pem should exist" - assert "openshell-tls" in env["SSL_CERT_FILE"] - assert "openshell-tls" in env["NODE_EXTRA_CA_CERTS"] - - -def test_l7_tls_deny_response_format( - sandbox: Callable[..., Sandbox], -) -> None: - """L7-T6: L7 deny response is valid JSON with expected fields.""" - policy = _base_policy( - network_policies={ - "anthropic": sandbox_pb2.NetworkPolicyRule( - name="anthropic", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host="api.anthropic.com", - port=443, - protocol="rest", - tls="terminate", - enforcement="enforce", - access="read-only", - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python( - _proxy_connect_then_http(), - args=("api.anthropic.com", 443, "DELETE", "/v1/anything"), - ) - assert result.exit_code == 0, result.stderr - resp = json.loads(result.stdout) - assert resp["http_status"] == 403 - - # Verify response headers - assert "X-OpenShell-Policy" in resp["headers"] - assert "application/json" in resp["headers"] - - # Verify JSON body structure - body = json.loads(resp["body"]) - assert body["error"] == "policy_denied" - assert "policy" in body - assert "rule" in body - assert "detail" in body - - -def test_l7_tls_log_fields( - sandbox: Callable[..., Sandbox], -) -> None: - """L7-T7: L7 request logging includes structured fields.""" - policy = _base_policy( - network_policies={ - "anthropic": sandbox_pb2.NetworkPolicyRule( - name="anthropic", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host="api.anthropic.com", - port=443, - protocol="rest", - tls="terminate", - enforcement="enforce", - access="full", - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - sb.exec_python( - _proxy_connect_then_http(), - args=("api.anthropic.com", 443, "GET", "/v1/models"), - ) - - log_result = sb.exec_python(_read_openshell_log()) - assert log_result.exit_code == 0, log_result.stderr - log = log_result.stdout - - # OCSF shorthand: L7 requests show as HTTP:method events - assert "HTTP:" in log, "Expected OCSF HTTP activity event in log" - assert "ALLOWED" in log or "DENIED" in log, "Expected L7 decision in log" - assert "policy:" in log, "Expected policy context in log" - - -def test_l7_query_matchers_enforced( - sandbox: Callable[..., Sandbox], -) -> None: - """L7-T9: Query matcher glob/any allows and denies as expected.""" - policy = _base_policy( - network_policies={ - "query_api": sandbox_pb2.NetworkPolicyRule( - name="query_api", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host=_SANDBOX_IP, - port=_FORWARD_PROXY_PORT, - protocol="rest", - enforcement="enforce", - allowed_ips=["10.200.0.0/24"], - rules=[ - sandbox_pb2.L7Rule( - allow=sandbox_pb2.L7Allow( - method="GET", - path="/download", - query={ - "tag": sandbox_pb2.L7QueryMatcher(glob="foo-*"), - }, - ), - ), - sandbox_pb2.L7Rule( - allow=sandbox_pb2.L7Allow( - method="GET", - path="/search", - query={ - "tag": sandbox_pb2.L7QueryMatcher( - any=["foo-*", "bar-*"] - ), - }, - ), - ), - ], - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - allowed = sb.exec_python( - _proxy_connect_then_http_with_server(), - args=( - _PROXY_HOST, - _PROXY_PORT, - _SANDBOX_IP, - _FORWARD_PROXY_PORT, - "GET", - "/download?tag=foo-a&tag=foo-b", - ), - ) - assert allowed.exit_code == 0, allowed.stderr - allowed_resp = json.loads(allowed.stdout) - assert "200" in allowed_resp["connect_status"] - assert allowed_resp["http_status"] == 200 - assert "connect-server-ok" in allowed_resp["body"] - - denied = sb.exec_python( - _proxy_connect_then_http_with_server(), - args=( - _PROXY_HOST, - _PROXY_PORT, - _SANDBOX_IP, - _FORWARD_PROXY_PORT, - "GET", - "/download?tag=foo-a&tag=evil", - ), - ) - assert denied.exit_code == 0, denied.stderr - denied_resp = json.loads(denied.stdout) - assert denied_resp["http_status"] == 403 - assert "policy_denied" in denied_resp["body"] - - any_allowed = sb.exec_python( - _proxy_connect_then_http_with_server(), - args=( - _PROXY_HOST, - _PROXY_PORT, - _SANDBOX_IP, - _FORWARD_PROXY_PORT, - "GET", - "/search?tag=foo-a&tag=bar-b", - ), - ) - assert any_allowed.exit_code == 0, any_allowed.stderr - any_resp = json.loads(any_allowed.stdout) - assert any_resp["http_status"] == 200 - assert "connect-server-ok" in any_resp["body"] - - missing_required = sb.exec_python( - _proxy_connect_then_http_with_server(), - args=( - _PROXY_HOST, - _PROXY_PORT, - _SANDBOX_IP, - _FORWARD_PROXY_PORT, - "GET", - "/download?slug=skill-1", - ), - ) - assert missing_required.exit_code == 0, missing_required.stderr - missing_resp = json.loads(missing_required.stdout) - assert missing_resp["http_status"] == 403 - assert "policy_denied" in missing_resp["body"] - - -def test_l7_rule_without_query_matcher_allows_any_query_params( - sandbox: Callable[..., Sandbox], -) -> None: - """L7-T10: Rule without query matcher allows any query params.""" - policy = _base_policy( - network_policies={ - "query_optional": sandbox_pb2.NetworkPolicyRule( - name="query_optional", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host=_SANDBOX_IP, - port=_FORWARD_PROXY_PORT, - protocol="rest", - enforcement="enforce", - allowed_ips=["10.200.0.0/24"], - rules=[ - sandbox_pb2.L7Rule( - allow=sandbox_pb2.L7Allow( - method="GET", - path="/download", - ), - ), - ], - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python( - _proxy_connect_then_http_with_server(), - args=( - _PROXY_HOST, - _PROXY_PORT, - _SANDBOX_IP, - _FORWARD_PROXY_PORT, - "GET", - "/download?tag=anything&slug=any-value", - ), - ) - assert result.exit_code == 0, result.stderr - resp = json.loads(result.stdout) - assert "200" in resp["connect_status"] - assert resp["http_status"] == 200 - assert "connect-server-ok" in resp["body"] - - -# ============================================================================= -# Forward proxy tests (plain HTTP, non-CONNECT) -# ============================================================================= - -# The sandbox's own IP within the network namespace -_SANDBOX_IP = "10.200.0.2" -_FORWARD_PROXY_PORT = 19876 - - -def test_forward_proxy_allows_private_ip_with_allowed_ips( - sandbox: Callable[..., Sandbox], -) -> None: - """FWD-1: Forward proxy GET to private IP with allowed_ips succeeds. - - Starts an HTTP server inside the sandbox, sends a plain forward proxy - request through the sandbox proxy, and verifies the response is relayed. - """ - policy = _base_policy( - network_policies={ - "internal_http": sandbox_pb2.NetworkPolicyRule( - name="internal_http", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host=_SANDBOX_IP, - port=_FORWARD_PROXY_PORT, - allowed_ips=["10.200.0.0/24"], - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python( - _forward_proxy_with_server(), - args=(_PROXY_HOST, _PROXY_PORT, _SANDBOX_IP, _FORWARD_PROXY_PORT), - ) - assert result.exit_code == 0, result.stderr - assert "200" in result.stdout, ( - f"Expected 200 in forward proxy response, got: {result.stdout}" - ) - assert "forward-proxy-ok" in result.stdout, ( - f"Expected response body relayed, got: {result.stdout}" - ) - - -def test_forward_proxy_allows_private_ip_host_without_allowed_ips( - sandbox: Callable[..., Sandbox], -) -> None: - """FWD-2: Forward proxy to literal IP host without allowed_ips -> 200. - - When the policy host field is a literal IP address, the user has explicitly - declared intent to allow that destination. The SSRF guard synthesizes an - implicit allowed_ips entry, so explicit allowed_ips is not required. - """ - policy = _base_policy( - network_policies={ - "internal_http": sandbox_pb2.NetworkPolicyRule( - name="internal_http", - endpoints=[ - # No allowed_ips — but host is a literal IP, so implicit - sandbox_pb2.NetworkEndpoint( - host=_SANDBOX_IP, - port=_FORWARD_PROXY_PORT, - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python( - _forward_proxy_with_server(), - args=(_PROXY_HOST, _PROXY_PORT, _SANDBOX_IP, _FORWARD_PROXY_PORT), - ) - assert result.exit_code == 0, result.stderr - assert "200" in result.stdout, ( - f"Expected 200 for literal IP host, got: {result.stdout}" - ) - assert "forward-proxy-ok" in result.stdout, ( - f"Expected response body relayed, got: {result.stdout}" - ) - - -def test_forward_proxy_rejects_https_scheme( - sandbox: Callable[..., Sandbox], -) -> None: - """FWD-3: Forward proxy with https:// scheme -> 400. - - HTTPS must use CONNECT tunneling, not forward proxy. - """ - policy = _base_policy( - network_policies={ - "internal_http": sandbox_pb2.NetworkPolicyRule( - name="internal_http", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host=_SANDBOX_IP, - port=_FORWARD_PROXY_PORT, - allowed_ips=["10.200.0.0/24"], - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python( - _forward_proxy_raw(), - args=( - _PROXY_HOST, - _PROXY_PORT, - f"https://{_SANDBOX_IP}:{_FORWARD_PROXY_PORT}/test", - ), - ) - assert result.exit_code == 0, result.stderr - assert "400" in result.stdout, ( - f"Expected 400 for HTTPS forward proxy, got: {result.stdout}" - ) - - -def test_forward_proxy_denied_no_policy_match( - sandbox: Callable[..., Sandbox], -) -> None: - """FWD-4: Forward proxy to unmatched host:port -> 403.""" - policy = _base_policy( - network_policies={ - "other": sandbox_pb2.NetworkPolicyRule( - name="other", - endpoints=[ - # Policy for a different host/port - sandbox_pb2.NetworkEndpoint( - host="10.200.0.1", - port=9999, - allowed_ips=["10.200.0.0/24"], - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python( - _forward_proxy_raw(), - args=( - _PROXY_HOST, - _PROXY_PORT, - f"http://{_SANDBOX_IP}:{_FORWARD_PROXY_PORT}/test", - ), - ) - assert result.exit_code == 0, result.stderr - assert "403" in result.stdout, ( - f"Expected 403 for unmatched policy, got: {result.stdout}" - ) - - -def test_forward_proxy_public_ip_denied( - sandbox: Callable[..., Sandbox], -) -> None: - """FWD-5: Forward proxy to public IP -> 403. - - Even with allowed_ips, forward proxy is restricted to private IPs. - Plain HTTP should never traverse the public internet. - """ - policy = _base_policy( - network_policies={ - "public": sandbox_pb2.NetworkPolicyRule( - name="public", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host="example.com", - port=80, - allowed_ips=["93.184.0.0/16"], - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python( - _forward_proxy_raw(), - args=(_PROXY_HOST, _PROXY_PORT, "http://example.com/"), - ) - assert result.exit_code == 0, result.stderr - assert "403" in result.stdout, ( - f"Expected 403 for public IP forward proxy, got: {result.stdout}" - ) - - -def test_forward_proxy_log_fields( - sandbox: Callable[..., Sandbox], -) -> None: - """FWD-6: Forward proxy requests produce structured FORWARD log lines.""" - policy = _base_policy( - network_policies={ - "internal_http": sandbox_pb2.NetworkPolicyRule( - name="internal_http", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host=_SANDBOX_IP, - port=_FORWARD_PROXY_PORT, - allowed_ips=["10.200.0.0/24"], - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - # Trigger an allowed forward proxy request (with server) - sb.exec_python( - _forward_proxy_with_server(), - args=(_PROXY_HOST, _PROXY_PORT, _SANDBOX_IP, _FORWARD_PROXY_PORT), - ) - # Trigger a denied forward proxy request (no allowed_ips match) - sb.exec_python( - _forward_proxy_raw(), - args=( - _PROXY_HOST, - _PROXY_PORT, - "http://example.com/", - ), - ) - # Read the log - result = sb.exec_python(_read_openshell_log()) - assert result.exit_code == 0, result.stderr - log = result.stdout - - # OCSF shorthand: FORWARD requests show as HTTP:method events - assert "HTTP:" in log, "Expected OCSF HTTP activity event for FORWARD request" - assert "ALLOWED" in log, "Expected ALLOWED for forward proxy allow" - assert f"{_SANDBOX_IP}" in log, "Expected destination IP in FORWARD log" - - -# ============================================================================= -# Baseline filesystem path enrichment tests (BFS-*) -# ============================================================================= - - -def _verify_sandbox_functional(): - """Return a closure that verifies basic sandbox functionality.""" - - def fn(): - import json - import os - import sys - - checks = {} - # Can resolve DNS config - checks["resolv_conf"] = os.path.exists("/etc/resolv.conf") - # Can access shared libraries - checks["lib_exists"] = os.path.isdir("/usr/lib") - # Python interpreter works - checks["python_version"] = sys.version - # Can write to /tmp - tmp_path = "/tmp/enrichment_test.txt" - try: - with open(tmp_path, "w") as f: - f.write("ok") - with open(tmp_path) as f: - checks["tmp_write"] = f.read() == "ok" - os.unlink(tmp_path) - except Exception as e: - checks["tmp_write"] = str(e) - # Can write to /sandbox - sb_path = "/sandbox/enrichment_test.txt" - try: - with open(sb_path, "w") as f: - f.write("ok") - with open(sb_path) as f: - checks["sandbox_write"] = f.read() == "ok" - os.unlink(sb_path) - except Exception as e: - checks["sandbox_write"] = str(e) - # Can read openshell log (rolling appender writes date-stamped files) - import glob - - checks["var_log"] = len(glob.glob("/var/log/openshell*.log*")) > 0 - return json.dumps(checks) - - return fn - - -def test_baseline_enrichment_missing_filesystem_policy( - sandbox: Callable[..., Sandbox], -) -> None: - """BFS-1: Sandbox with network_policies but NO filesystem_policy should - come up and function correctly thanks to baseline path enrichment.""" - # Intentionally omit filesystem, landlock, and process fields — - # only provide network_policies. - spec = datamodel_pb2.SandboxSpec( - policy=sandbox_pb2.SandboxPolicy( - version=1, - network_policies={ - "test": sandbox_pb2.NetworkPolicyRule( - name="test", - endpoints=[ - sandbox_pb2.NetworkEndpoint(host="example.com", port=443), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ), - ) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python(_verify_sandbox_functional()) - assert result.exit_code == 0, ( - f"Sandbox with missing filesystem_policy failed to run: {result.stderr}" - ) - import json - - checks = json.loads(result.stdout) - assert checks["resolv_conf"] is True, "DNS config not accessible" - assert checks["lib_exists"] is True, "Shared libraries not accessible" - assert checks["tmp_write"] is True, f"/tmp not writable: {checks['tmp_write']}" - assert checks["sandbox_write"] is True, ( - f"/sandbox not writable: {checks['sandbox_write']}" - ) - assert checks["var_log"] is True, "OpenShell log not accessible" - - -def test_baseline_enrichment_incomplete_filesystem_policy( - sandbox: Callable[..., Sandbox], -) -> None: - """BFS-2: Sandbox with filesystem_policy that only has /sandbox should - still function because baseline enrichment adds missing paths.""" - spec = datamodel_pb2.SandboxSpec( - policy=sandbox_pb2.SandboxPolicy( - version=1, - filesystem=sandbox_pb2.FilesystemPolicy( - include_workdir=True, - read_only=[], - read_write=["/sandbox"], - ), - landlock=sandbox_pb2.LandlockPolicy(compatibility="best_effort"), - process=sandbox_pb2.ProcessPolicy( - run_as_user="sandbox", - run_as_group="sandbox", - ), - network_policies={ - "test": sandbox_pb2.NetworkPolicyRule( - name="test", - endpoints=[ - sandbox_pb2.NetworkEndpoint(host="example.com", port=443), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ), - ) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python(_verify_sandbox_functional()) - assert result.exit_code == 0, ( - f"Sandbox with incomplete filesystem_policy failed to run: {result.stderr}" - ) - import json - - checks = json.loads(result.stdout) - assert checks["resolv_conf"] is True, "DNS config not accessible" - assert checks["lib_exists"] is True, "Shared libraries not accessible" - assert checks["tmp_write"] is True, f"/tmp not writable: {checks['tmp_write']}" - assert checks["sandbox_write"] is True, ( - f"/sandbox not writable: {checks['sandbox_write']}" - ) - assert checks["var_log"] is True, "OpenShell log not accessible" - - -# ============================================================================= -# Multi-port endpoint tests -# ============================================================================= -# -# MP-1: Multi-port endpoint allows connections on any listed port -# MP-2: Multi-port endpoint denies connections on unlisted ports -# MP-3: Single port (backwards compat) still works via ports normalization -# ============================================================================= - - -def test_multi_port_allows_all_listed_ports( - sandbox: Callable[..., Sandbox], -) -> None: - """MP-1: Multi-port endpoint allows connections on any listed port. - - Policy allows python -> api.anthropic.com on ports 443 AND 80. - Both should be allowed; port 8080 should be denied. - """ - policy = _base_policy( - network_policies={ - "multi": sandbox_pb2.NetworkPolicyRule( - name="multi", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host="api.anthropic.com", ports=[443, 80] - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - # Port 443 -> allowed - result = sb.exec_python(_proxy_connect(), args=("api.anthropic.com", 443)) - assert result.exit_code == 0, result.stderr - assert "200" in result.stdout, f"Port 443 should be allowed: {result.stdout}" - - # Port 80 -> allowed - result = sb.exec_python(_proxy_connect(), args=("api.anthropic.com", 80)) - assert result.exit_code == 0, result.stderr - assert "200" in result.stdout, f"Port 80 should be allowed: {result.stdout}" - - -def test_multi_port_denies_unlisted_port( - sandbox: Callable[..., Sandbox], -) -> None: - """MP-2: Multi-port endpoint denies connections on ports not in the list.""" - policy = _base_policy( - network_policies={ - "multi": sandbox_pb2.NetworkPolicyRule( - name="multi", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host="api.anthropic.com", ports=[443, 80] - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - # Port 8080 -> denied (not in [443, 80]) - result = sb.exec_python(_proxy_connect(), args=("api.anthropic.com", 8080)) - assert result.exit_code == 0, result.stderr - assert "403" in result.stdout, f"Port 8080 should be denied: {result.stdout}" - - -def test_single_port_backwards_compat( - sandbox: Callable[..., Sandbox], -) -> None: - """MP-3: Old-style single port field still works.""" - policy = _base_policy( - network_policies={ - "compat": sandbox_pb2.NetworkPolicyRule( - name="compat", - endpoints=[ - sandbox_pb2.NetworkEndpoint(host="api.anthropic.com", port=443), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - # Port 443 -> allowed - result = sb.exec_python(_proxy_connect(), args=("api.anthropic.com", 443)) - assert result.exit_code == 0, result.stderr - assert "200" in result.stdout, f"Single port should still work: {result.stdout}" - - # Port 80 -> denied - result = sb.exec_python(_proxy_connect(), args=("api.anthropic.com", 80)) - assert result.exit_code == 0, result.stderr - assert "403" in result.stdout - - -# ============================================================================= -# Host wildcard tests -# ============================================================================= -# -# HW-1: Wildcard host pattern matches subdomains -# HW-2: Wildcard host pattern does NOT match the bare domain -# HW-3: Wildcard host pattern does NOT match deep subdomains -# ============================================================================= - - -def test_host_wildcard_matches_subdomain( - sandbox: Callable[..., Sandbox], -) -> None: - """HW-1: host wildcard matches single-label subdomains.""" - policy = _base_policy( - network_policies={ - "wildcard": sandbox_pb2.NetworkPolicyRule( - name="wildcard", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host=_PUBLIC_WILDCARD_PATTERN, - port=443, - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python( - _proxy_connect(), args=(_PUBLIC_WILDCARD_SUBDOMAIN, 443) - ) - assert result.exit_code == 0, result.stderr - assert "200" in result.stdout, ( - f"{_PUBLIC_WILDCARD_PATTERN} should match " - f"{_PUBLIC_WILDCARD_SUBDOMAIN}: " - f"{result.stdout}" - ) - - # example.com -> does NOT match the wildcard pattern - result = sb.exec_python(_proxy_connect(), args=("example.com", 443)) - assert result.exit_code == 0, result.stderr - assert "403" in result.stdout, ( - f"{_PUBLIC_WILDCARD_PATTERN} should NOT match example.com: " - f"{result.stdout}" - ) - - -def test_host_wildcard_rejects_bare_domain( - sandbox: Callable[..., Sandbox], -) -> None: - """HW-2: host wildcard does NOT match the bare domain.""" - policy = _base_policy( - network_policies={ - "wildcard": sandbox_pb2.NetworkPolicyRule( - name="wildcard", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host=_PUBLIC_WILDCARD_PATTERN, - port=443, - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - result = sb.exec_python(_proxy_connect(), args=(_PUBLIC_WILDCARD_SUFFIX, 443)) - assert result.exit_code == 0, result.stderr - assert "403" in result.stdout, ( - f"{_PUBLIC_WILDCARD_PATTERN} should NOT match bare " - f"{_PUBLIC_WILDCARD_SUFFIX}: {result.stdout}" - ) - - -def test_host_wildcard_rejects_deep_subdomain( - sandbox: Callable[..., Sandbox], -) -> None: - """HW-3: host wildcard does NOT match a deep subdomain. - - Single * matches one DNS label only (does not cross . boundaries). - """ - policy = _base_policy( - network_policies={ - "wildcard": sandbox_pb2.NetworkPolicyRule( - name="wildcard", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host=_PUBLIC_WILDCARD_PATTERN, - port=443, - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - deep_subdomain = f"deep.sub.{_PUBLIC_WILDCARD_SUFFIX}" - result = sb.exec_python(_proxy_connect(), args=(deep_subdomain, 443)) - assert result.exit_code == 0, result.stderr - assert "403" in result.stdout, ( - f"{_PUBLIC_WILDCARD_PATTERN} should NOT match {deep_subdomain}: " - f"{result.stdout}" - ) - - -# ============================================================================= -# Overlapping policies (duplicate host:port) — regression tests -# ============================================================================= - - -def test_overlapping_policies_with_conflicting_destination_metadata_are_rejected( - sandbox: Callable[..., Sandbox], -) -> None: - """OVL-1: Conflicting metadata on the same host:port fails closed. - - One endpoint permits any resolved address while the other constrains - ``allowed_ips``. The complete candidate is ambiguous and must be rejected - before the sandbox is provisioned. - """ - policy = _base_policy( - network_policies={ - "user_rule": sandbox_pb2.NetworkPolicyRule( - name="user_rule", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host=_SANDBOX_IP, - port=_FORWARD_PROXY_PORT, - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - "approved_rule": sandbox_pb2.NetworkPolicyRule( - name="approved_rule", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host=_SANDBOX_IP, - port=_FORWARD_PROXY_PORT, - allowed_ips=["10.200.0.0/24"], - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with ( - pytest.raises(grpc.RpcError) as exc_info, - sandbox(spec=spec, delete_on_exit=True), - ): - pytest.fail("ambiguous policy unexpectedly created a sandbox") + with ( + pytest.raises(grpc.RpcError) as exc_info, + sandbox(spec=spec, delete_on_exit=True), + ): + pytest.fail("ambiguous policy unexpectedly created a sandbox") assert exc_info.value.code() == grpc.StatusCode.FAILED_PRECONDITION details = exc_info.value.details() or "" assert "network endpoint ambiguity validation failed" in details assert "allowed_ips" in details - - -def test_overlapping_policies_l7_connect_does_not_crash( - sandbox: Callable[..., Sandbox], -) -> None: - """OVL-2: CONNECT to overlapping L7 policies must not crash OPA. - - Two policies with L7 rules (protocol: rest) covering the same host:port - must evaluate without a regorus variable collision error. - """ - policy = _base_policy( - network_policies={ - "user_api": sandbox_pb2.NetworkPolicyRule( - name="user_api", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host="api.anthropic.com", - port=443, - protocol="rest", - enforcement="enforce", - access="read-only", - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - "auto_approved_api": sandbox_pb2.NetworkPolicyRule( - name="auto_approved_api", - endpoints=[ - sandbox_pb2.NetworkEndpoint( - host="api.anthropic.com", - port=443, - protocol="rest", - enforcement="enforce", - access="read-only", - ), - ], - binaries=[sandbox_pb2.NetworkBinary(path="/**")], - ), - }, - ) - spec = datamodel_pb2.SandboxSpec(policy=policy) - with sandbox(spec=spec, delete_on_exit=True) as sb: - # CONNECT should succeed at the tunnel level (200 Connection Established) - # even with two overlapping L7 policies. - result = sb.exec_python(_proxy_connect(), args=("api.anthropic.com", 443)) - assert result.exit_code == 0, result.stderr - assert "200" in result.stdout, ( - f"Overlapping L7 policies should not crash; expected 200, got: {result.stdout}" - ) diff --git a/e2e/python/test_sandbox_venv.py b/e2e/python/test_sandbox_venv.py index 10d2d4d746..0461542e0d 100644 --- a/e2e/python/test_sandbox_venv.py +++ b/e2e/python/test_sandbox_venv.py @@ -5,32 +5,68 @@ Verifies that: - /sandbox/.venv/bin is in PATH for both interactive and non-interactive sessions -- pip install works inside the sandbox (pypi policy in dev-sandbox-policy.yaml) +- pip install works inside the sandbox with an explicit PyPI policy - uv pip install works (validates Landlock V2 cross-directory rename support) - uv run --with works for ephemeral dependency injection - Installed packages are importable after installation -All tests use the default dev sandbox policy -- no custom policy overrides. -The SDK omits the policy field from the spec so the sandbox container discovers -its policy from /etc/openshell/policy.yaml (the dev-sandbox-policy.yaml baked -into the image), which already includes the pypi network policy. +Package-install tests pass their policy explicitly. In the split sandbox and +supervisor topology, trusted policy evaluation no longer discovers policy from +the untrusted workload image filesystem. """ from __future__ import annotations from typing import TYPE_CHECKING +from openshell._proto import datamodel_pb2, sandbox_pb2 + if TYPE_CHECKING: from collections.abc import Callable from openshell import Sandbox +def _pypi_spec() -> datamodel_pb2.SandboxSpec: + endpoints = [ + "pypi.org", + "files.pythonhosted.org", + "github.com", + "objects.githubusercontent.com", + "api.github.com", + "downloads.python.org", + ] + return datamodel_pb2.SandboxSpec( + policy=sandbox_pb2.SandboxPolicy( + version=1, + filesystem=sandbox_pb2.FilesystemPolicy( + include_workdir=True, + read_only=["/usr", "/lib", "/etc", "/app", "/proc"], + read_write=["/sandbox", "/tmp"], + ), + landlock=sandbox_pb2.LandlockPolicy(compatibility="best_effort"), + process=sandbox_pb2.ProcessPolicy( + run_as_user="sandbox", run_as_group="sandbox" + ), + network_policies={ + "pypi": sandbox_pb2.NetworkPolicyRule( + name="pypi", + endpoints=[ + sandbox_pb2.NetworkEndpoint(host=host, port=443) + for host in endpoints + ], + binaries=[sandbox_pb2.NetworkBinary(path="/**")], + ) + }, + ) + ) + + def test_sandbox_venv_in_path( sandbox: Callable[..., Sandbox], ) -> None: """Non-interactive exec sees /sandbox/.venv/bin in PATH.""" - with sandbox(delete_on_exit=True) as sb: + with sandbox(spec=_pypi_spec(), delete_on_exit=True) as sb: result = sb.exec(["bash", "-c", "echo $PATH"], timeout_seconds=20) assert result.exit_code == 0, result.stderr path_dirs = result.stdout.strip().split(":") @@ -43,7 +79,7 @@ def test_pip_install_in_sandbox( sandbox: Callable[..., Sandbox], ) -> None: """pip install works inside the sandbox and installed packages are importable.""" - with sandbox(delete_on_exit=True) as sb: + with sandbox(spec=_pypi_spec(), delete_on_exit=True) as sb: install = sb.exec( ["pip", "install", "--quiet", "cowsay"], timeout_seconds=60, @@ -72,7 +108,7 @@ def test_uv_pip_install_in_sandbox( because uv uses cross-directory rename() for cache population and installation. Landlock V2 adds the REFER right which permits this. """ - with sandbox(delete_on_exit=True) as sb: + with sandbox(spec=_pypi_spec(), delete_on_exit=True) as sb: install = sb.exec( [ "uv", @@ -105,7 +141,7 @@ def test_uv_run_with_ephemeral_dependency( sandbox: Callable[..., Sandbox], ) -> None: """uv run --with installs a dependency on-the-fly and runs a script using it.""" - with sandbox(delete_on_exit=True) as sb: + with sandbox(spec=_pypi_spec(), delete_on_exit=True) as sb: result = sb.exec( [ "uv", diff --git a/e2e/rust/tests/credential_gating.rs b/e2e/rust/tests/credential_gating.rs index 3fbbf6b21b..2afe9ec231 100644 --- a/e2e/rust/tests/credential_gating.rs +++ b/e2e/rust/tests/credential_gating.rs @@ -221,6 +221,7 @@ enum EndpointMode { TlsSkip, L4OptIn, RestBody { rewrite: bool }, + WebSocket, } #[derive(Clone, Copy)] @@ -244,6 +245,9 @@ fn write_policy( EndpointMode::RestBody { rewrite } => format!( " protocol: rest\n access: full\n request_body_credential_rewrite: {rewrite}\n" ), + EndpointMode::WebSocket => { + " protocol: websocket\n access: read-write\n".to_string() + } }; let credential_binding = match credential_source { CredentialSource::ProviderProfile => String::new(), @@ -268,12 +272,7 @@ network_policies: endpoints: - host: {TEST_HOST} port: {port} -{endpoint_options}{credential_binding} allowed_ips: - - "10.0.0.0/8" - - "172.0.0.0/8" - - "192.168.0.0/16" - - "fc00::/7" - binaries: +{endpoint_options}{credential_binding} binaries: - path: /usr/bin/python* - path: /usr/local/bin/python* - path: /sandbox/.uv/python/*/bin/python* @@ -286,32 +285,10 @@ network_policies: Ok(file) } -fn write_base_policy() -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - file.write_all( - br#"version: 1 -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] -landlock: - compatibility: best_effort -process: - run_as_user: sandbox - run_as_group: sandbox -"#, - ) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - #[derive(Debug, Default, Clone, Copy)] struct BodyObservation { saw_placeholder: bool, saw_secret: bool, - authenticated: bool, } struct HttpProbeServer { @@ -529,20 +506,11 @@ async fn handle_http_probe( } } - let header_end = received - .windows(4) - .position(|w| w == b"\r\n\r\n") - .map_or(received.len(), |end| end + 4); - let headers = String::from_utf8_lossy(&received[..header_end]); - let body = &received[header_end..]; let observation = BodyObservation { - authenticated: headers - .lines() - .any(|line| line == format!("Authorization: Bearer {TEST_SECRET}")), - saw_placeholder: body + saw_placeholder: received .windows(PLACEHOLDER_PREFIX.len()) .any(|window| window == PLACEHOLDER_PREFIX.as_bytes()), - saw_secret: body + saw_secret: received .windows(TEST_SECRET.len()) .any(|window| window == TEST_SECRET.as_bytes()), }; @@ -550,23 +518,14 @@ async fn handle_http_probe( if expected_total.is_some_and(|expected| received.len() >= expected) { let result = if observation.saw_secret && !observation.saw_placeholder { "BODY_REWRITTEN" - } else if observation.saw_placeholder && !observation.saw_secret { - "BODY_TEXT" } else { "BODY_BAD" }; - // Echo admitted literal bodies so clients can assert exact preservation. - let response_body = if result == "BODY_TEXT" { - [b"BODY_TEXT\n".as_slice(), body].concat() - } else { - result.as_bytes().to_vec() - }; let response = format!( - "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - response_body.len() + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{result}", + result.len() ); stream.write_all(response.as_bytes()).await?; - stream.write_all(&response_body).await?; } Ok(()) } @@ -599,7 +558,7 @@ with socket.create_connection((host, port), timeout=10) as sock: if not chunk: break response += chunk - print("BODY_REWRITTEN" if b"BODY_REWRITTEN" in response else "BODY_TEXT" if b"BODY_TEXT" in response else "BODY_DENIED") + print("BODY_REWRITTEN" if b"BODY_REWRITTEN" in response else "BODY_DENIED") "# ) } @@ -854,55 +813,17 @@ async fn run_body_sandbox( Ok(output) } -async fn assert_conversation_placeholders_pass( - server: &HttpProbeServer, - own_provider: bool, -) -> Result<(), String> { - let policy = if own_provider { - write_base_policy()? - } else { - write_policy( - server.port, - EndpointMode::RestBody { rewrite: false }, - CredentialSource::ProviderProfile, - )? - }; - let policy_path = policy.path().to_str().ok_or("invalid policy path")?; - let script = format!( - r#" -import http.client -import json -import os -import urllib.parse - -proxy_url = next(os.environ[name] for name in - ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy") - if os.environ.get(name)) -proxy = urllib.parse.urlparse(proxy_url) -target = "{host}:{port}" -issued = os.environ[{env:?}] -for token in ("openshell:resolve:env:KEY", issued): - body = json.dumps({{"messages": [{{"role": "tool", "content": "{env}=" + token}}, {{"role": "user", "content": "hi"}}]}}).encode() - for tunnel in (False, True): - for replay in range(2): - connection = http.client.HTTPConnection(proxy.hostname, proxy.port or 80, timeout=10) - if tunnel: - connection.set_tunnel({host:?}, {port}) - headers = {{"Content-Type": "application/json", "Connection": "close"}} - if {own}: - headers["Authorization"] = "Bearer " + issued - connection.request("POST", "/token" if tunnel else "http://" + target + "/token", body, headers) - response = connection.getresponse() - assert response.status == 200, response.status - assert response.read() == b"BODY_TEXT\n" + body, "body changed" - connection.close() - print("BODY_TEXT") -"#, - host = TEST_HOST, - port = server.port, - env = TOKEN_ENV, - own = if own_provider { "True" } else { "False" }, - ); +async fn run_profile_body_sandbox(port: u16) -> Result { + let policy = write_policy( + port, + EndpointMode::RestBody { rewrite: false }, + CredentialSource::ProviderProfile, + )?; + let policy_path = policy + .path() + .to_str() + .ok_or_else(|| "body policy path is not UTF-8".to_string())?; + let script = body_client_script(port); let mut sandbox = SandboxGuard::create(&[ "--policy", policy_path, @@ -916,26 +837,25 @@ for token in ("openshell:resolve:env:KEY", issued): .await?; let output = sandbox.create_output.clone(); sandbox.cleanup().await; - assert_eq!( - output.matches("BODY_TEXT").count(), - 8, - "conversation did not survive replay: {output}" - ); - assert!(!output.contains(TEST_SECRET)); - let observations = server.wait_for_observations(8).await; - assert_eq!(observations.len(), 8); - assert!( - observations - .iter() - .all(|observation| observation.saw_placeholder - && !observation.saw_secret - && observation.authenticated == own_provider) - ); + Ok(output) +} + +async fn assert_rest_body_backstop(server: &HttpProbeServer) -> Result<(), String> { + let denied = run_profile_body_sandbox(server.port).await?; + assert!(denied.contains("BODY_DENIED")); + let observations = server.wait_for_observations(1).await; + assert_eq!(observations.len(), 1, "observations: {observations:?}"); + assert!(!observations[0].saw_placeholder); + assert!(!observations[0].saw_secret); Ok(()) } async fn assert_websocket_binary_denied(server: &BinaryWebSocketProbeServer) -> Result<(), String> { - let policy = write_base_policy()?; + let policy = write_policy( + server.port, + EndpointMode::WebSocket, + CredentialSource::ProviderProfile, + )?; let policy_path = policy .path() .to_str() @@ -976,9 +896,7 @@ async fn credentialed_endpoint_gates_work_end_to_end() { .expect("install credentialed provider"); let result = async { - assert_conversation_placeholders_pass(&server, true).await?; - let foreign_server = HttpProbeServer::start().await?; - assert_conversation_placeholders_pass(&foreign_server, false).await?; + assert_rest_body_backstop(&server).await?; assert_websocket_binary_denied(&websocket_server).await } .await; @@ -991,13 +909,13 @@ async fn credentialed_endpoint_gates_work_end_to_end() { .expect("install endpointless provider"); let endpointless_result = async { assert_gateway_admission(server.port, CredentialSource::PolicyBinding).await?; - let literal = run_body_sandbox( + let denied = run_body_sandbox( server.port, EndpointMode::RestBody { rewrite: false }, CredentialSource::PolicyBinding, ) .await?; - assert!(literal.contains("BODY_TEXT")); + assert!(denied.contains("BODY_DENIED")); let rewritten = run_body_sandbox( server.port, EndpointMode::RestBody { rewrite: true }, @@ -1007,12 +925,12 @@ async fn credentialed_endpoint_gates_work_end_to_end() { assert!(rewritten.contains("BODY_REWRITTEN")); assert!(!rewritten.contains(TEST_SECRET)); assert!(!rewritten.contains(PLACEHOLDER_PREFIX)); - let observations = server.wait_for_observations(10).await; - assert_eq!(observations.len(), 10, "observations: {observations:?}"); - assert!(observations[8].saw_placeholder); - assert!(!observations[8].saw_secret); - assert!(!observations[9].saw_placeholder); - assert!(observations[9].saw_secret); + let observations = server.wait_for_observations(3).await; + assert_eq!(observations.len(), 3, "observations: {observations:?}"); + assert!(!observations[1].saw_placeholder); + assert!(!observations[1].saw_secret); + assert!(!observations[2].saw_placeholder); + assert!(observations[2].saw_secret); assert_endpointless_provider_env_live_update(server.port).await?; Ok::<(), String>(()) } diff --git a/e2e/rust/tests/driver_config_volume.rs b/e2e/rust/tests/driver_config_volume.rs index 262e057629..8aaf1e3cb6 100644 --- a/e2e/rust/tests/driver_config_volume.rs +++ b/e2e/rust/tests/driver_config_volume.rs @@ -249,10 +249,20 @@ fn write_bind_mount_policy() -> Result { let mut file = tempfile::NamedTempFile::new().map_err(|err| format!("create bind policy: {err}"))?; file.write_all( - br"version: 1 + br#"version: 1 filesystem_policy: include_workdir: false + read_only: + - "/bin" + - "/dev" + - "/etc" + - "/lib" + - "/proc" + - "/usr" + read_write: + - "/sandbox/e2e-bind" + - "/tmp" landlock: compatibility: best_effort @@ -260,7 +270,7 @@ landlock: process: run_as_user: sandbox run_as_group: sandbox -", +"#, ) .map_err(|err| format!("write bind policy: {err}"))?; Ok(file) diff --git a/e2e/rust/tests/forward_proxy_l7_bypass.rs b/e2e/rust/tests/forward_proxy_l7_bypass.rs index f5df4f53e3..e346ac169a 100644 --- a/e2e/rust/tests/forward_proxy_l7_bypass.rs +++ b/e2e/rust/tests/forward_proxy_l7_bypass.rs @@ -14,9 +14,7 @@ use openshell_e2e::harness::container::ContainerHttpServer; use openshell_e2e::harness::sandbox::SandboxGuard; use tempfile::NamedTempFile; -const TEST_SERVER_ALIAS: &str = "rest-l7.openshell.test"; - -async fn start_test_server() -> Result { +async fn start_test_server(alias: &str) -> Result { let script = r#"from http.server import BaseHTTPRequestHandler, HTTPServer class Handler(BaseHTTPRequestHandler): @@ -34,7 +32,7 @@ class Handler(BaseHTTPRequestHandler): HTTPServer(("0.0.0.0", 8000), Handler).serve_forever() "#; - ContainerHttpServer::start_python(TEST_SERVER_ALIAS, script).await + ContainerHttpServer::start_python(alias, script).await } fn write_policy_with_l7_rules(host: &str, port: u16) -> Result { @@ -98,7 +96,9 @@ network_policies: /// GET /allowed should succeed — the L7 policy explicitly allows it. #[tokio::test] async fn forward_proxy_allows_l7_permitted_request() { - let server = start_test_server().await.expect("start test server"); + let server = start_test_server("rest-l7-allow.openshell.test") + .await + .expect("start test server"); let policy = write_policy_with_l7_rules(&server.host, server.port).expect("write custom policy"); let policy_path = policy @@ -148,7 +148,9 @@ print(json.dumps(last)) /// POST /allowed should be denied — the L7 policy only allows GET. #[tokio::test] async fn forward_proxy_denies_l7_blocked_request() { - let server = start_test_server().await.expect("start test server"); + let server = start_test_server("rest-l7-deny.openshell.test") + .await + .expect("start test server"); let policy = write_policy_with_l7_rules(&server.host, server.port).expect("write custom policy"); let policy_path = policy diff --git a/e2e/rust/tests/gateway_start.rs b/e2e/rust/tests/gateway_start.rs index cca35e3d59..31ffabb003 100644 --- a/e2e/rust/tests/gateway_start.rs +++ b/e2e/rust/tests/gateway_start.rs @@ -26,12 +26,21 @@ const STOPPED_READY_MARKER: &str = "gateway-start-stopped-ready"; const START_FILE: &str = "/sandbox/gateway-start-state"; const SANDBOX_NAMESPACE_LABEL: &str = "openshell.ai/sandbox-namespace"; const SANDBOX_NAME_LABEL: &str = "openshell.ai/sandbox-name"; +const SANDBOX_ROLE_LABEL_FILTER: &str = "label=openshell.ai/isolation-role=sandbox"; fn sandbox_container_id(namespace: &str, sandbox_name: &str) -> Result { let namespace_filter = format!("label={SANDBOX_NAMESPACE_LABEL}={namespace}"); let sandbox_name_filter = format!("label={SANDBOX_NAME_LABEL}={sandbox_name}"); let output = Command::new("docker") - .args(["ps", "-aq", "--filter", MANAGED_BY_LABEL_FILTER, "--filter"]) + .args([ + "ps", + "-aq", + "--filter", + MANAGED_BY_LABEL_FILTER, + "--filter", + SANDBOX_ROLE_LABEL_FILTER, + "--filter", + ]) .arg(namespace_filter) .args(["--filter"]) .arg(sandbox_name_filter) diff --git a/e2e/rust/tests/local_driver_token_restart.rs b/e2e/rust/tests/local_driver_token_restart.rs index 5223e3a704..5c9661d89c 100644 --- a/e2e/rust/tests/local_driver_token_restart.rs +++ b/e2e/rust/tests/local_driver_token_restart.rs @@ -67,6 +67,7 @@ impl LocalDriver { match self { Self::Docker => vec![ "label=openshell.ai/managed-by=openshell".to_string(), + "label=openshell.ai/isolation-role=sandbox".to_string(), format!("label=openshell.ai/sandbox-namespace={namespace}"), format!("label=openshell.ai/sandbox-name={sandbox_name}"), ], @@ -274,8 +275,24 @@ async fn stop_container_sandbox( sandbox_name: &str, ) -> Result<(), String> { let container_id = sandbox_container_id(engine, driver, namespace, sandbox_name)?; - let token = read_bootstrap_token(engine, &container_id)?; - require_non_expiring_token(&token, "local-driver bootstrap JWT")?; + if driver == LocalDriver::Docker { + run_engine( + engine, + &[ + "exec".to_string(), + container_id.clone(), + "sh".to_string(), + "-c".to_string(), + format!("test ! -r {CONTAINER_TOKEN_MOUNT_PATH}"), + ], + ) + .map_err(|error| { + format!("Docker sandbox workload must not be able to read its bootstrap JWT: {error}") + })?; + } else { + let token = read_bootstrap_token(engine, &container_id)?; + require_non_expiring_token(&token, "local-driver bootstrap JWT")?; + } run_engine(engine, &["stop".to_string(), container_id.clone()])?; wait_for_container_running(engine, &container_id, false, Duration::from_secs(60)).await diff --git a/e2e/rust/tests/proxy_egress_pipeline.rs b/e2e/rust/tests/proxy_egress_pipeline.rs index a2b9c49a3f..f841e509c1 100644 --- a/e2e/rust/tests/proxy_egress_pipeline.rs +++ b/e2e/rust/tests/proxy_egress_pipeline.rs @@ -3,16 +3,16 @@ #![cfg(feature = "e2e")] -//! E2E coverage for the shared explicit-proxy egress pipeline. +//! E2E coverage for the transparent sandbox egress pipeline. //! -//! These tests exercise behavior that must remain identical while CONNECT and -//! forward HTTP converge on shared authorization, destination, and relay -//! primitives: -//! - live policy reloads affect new requests through both adapters and close a -//! pre-existing CONNECT HTTP stream before its next request is forwarded; +//! Workloads connect directly to their requested destinations. Seccomp +//! notification diverts those sockets to the supervisor without proxy +//! environment variables or explicit CONNECT requests. These tests cover: +//! - live policy reloads affect new requests and close a pre-existing HTTP +//! stream before its next request is forwarded; //! - `tls: skip` selects a byte-transparent TCP relay; //! - provider placeholders in HTTP headers and opted-in REST bodies are -//! resolved through both adapters without appearing in test output. +//! resolved without appearing in test output. use std::io::{self, Error, ErrorKind, Write}; use std::process::Stdio; @@ -22,6 +22,7 @@ use std::sync::{ }; use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::container::{SupportContainer, e2e_network_name}; use openshell_e2e::harness::sandbox::SandboxGuard; use serde_json::Value; use tempfile::{Builder as TempFileBuilder, NamedTempFile}; @@ -359,9 +360,9 @@ network_policies: } fn write_ip_literal_success_policy( - ip: &str, - explicit_port: u16, - implicit_port: u16, + explicit_ip: &str, + implicit_ip: &str, + port: u16, ) -> Result { let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; let policy = format!( @@ -384,13 +385,14 @@ network_policies: name: destination_successes endpoints: - host: {ip} - port: {explicit_port} - allowed_ips: ["{ip}/32"] - - host: {ip} - port: {implicit_port} + port: {port} + allowed_ips: ["{explicit_ip}/32"] + - host: {implicit_ip} + port: {port} binaries: - path: "/**" -"# +"#, + ip = explicit_ip, ); file.write_all(policy.as_bytes()) .map_err(|error| format!("write policy: {error}"))?; @@ -749,26 +751,15 @@ async fn handle_credential_probe(mut stream: TcpStream) -> io::Result<()> { stream.write_all(response.as_bytes()).await } -fn proxy_status_script(host: &str, port: u16) -> String { +fn transparent_status_script(host: &str, port: u16) -> String { format!( r#" import json -import os import socket -import urllib.parse HOST = {host:?} PORT = {port} -def proxy_parts(): - proxy_url = next( - os.environ[name] - for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - if os.environ.get(name) - ) - parsed = urllib.parse.urlparse(proxy_url) - return parsed.hostname, parsed.port or 80 - def read_headers(sock): data = b"" while b"\r\n\r\n" not in data: @@ -782,44 +773,35 @@ def status(response): parts = response.split(None, 2) return int(parts[1]) if len(parts) > 1 else 0 -def forward_status(): - proxy_host, proxy_port = proxy_parts() - target = f"{{HOST}}:{{PORT}}" - with socket.create_connection((proxy_host, proxy_port), timeout=10) as sock: - sock.sendall( - f"GET http://{{target}}/forward HTTP/1.1\r\n" - f"Host: {{target}}\r\nConnection: close\r\n\r\n".encode() - ) - return status(read_headers(sock)) - -def connect_status(): - proxy_host, proxy_port = proxy_parts() +def request_status(path): target = f"{{HOST}}:{{PORT}}" - with socket.create_connection((proxy_host, proxy_port), timeout=10) as sock: - sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) - code = status(read_headers(sock)) - if code != 200: - return code - sock.sendall( - f"GET /connect HTTP/1.1\r\nHost: {{target}}\r\nConnection: close\r\n\r\n".encode() - ) - return status(read_headers(sock)) - -print(json.dumps({{"connect": connect_status(), "forward": forward_status()}}, sort_keys=True)) + try: + with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall( + f"GET {{path}} HTTP/1.1\r\n" + f"Host: {{target}}\r\nConnection: close\r\n\r\n".encode() + ) + return status(read_headers(sock)) + except OSError as error: + return {{"errno": error.errno, "error": repr(error)}} + +print(json.dumps({{ + "first": request_status("/first"), + "second": request_status("/second"), +}}, sort_keys=True)) "#, host = host, port = port, ) } -fn persistent_connect_script(host: &str, port: u16) -> String { +fn persistent_transparent_script(host: &str, port: u16) -> String { format!( r#" import json import os import socket import time -import urllib.parse HOST = {host:?} PORT = {port} @@ -827,15 +809,6 @@ READY = "/tmp/proxy-reload-ready" GO = "/tmp/proxy-reload-go" RESULT = "/tmp/proxy-reload-result" -def proxy_parts(): - proxy_url = next( - os.environ[name] - for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - if os.environ.get(name) - ) - parsed = urllib.parse.urlparse(proxy_url) - return parsed.hostname, parsed.port or 80 - def read_response(sock): data = b"" while b"\r\n\r\n" not in data: @@ -855,15 +828,11 @@ def read_response(sock): body += chunk return int(headers.split(None, 2)[1]) -proxy_host, proxy_port = proxy_parts() target = f"{{HOST}}:{{PORT}}" failed_closed = False second_status = 0 try: - with socket.create_connection((proxy_host, proxy_port), timeout=10) as sock: - sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) - if read_response(sock) != 200: - raise RuntimeError("initial CONNECT was denied") + with socket.create_connection((HOST, PORT), timeout=10) as sock: sock.sendall( f"GET /before-reload HTTP/1.1\r\nHost: {{target}}\r\nConnection: keep-alive\r\n\r\n".encode() ) @@ -920,7 +889,7 @@ fn parse_json_line(output: &str) -> Value { } #[tokio::test] -async fn policy_reload_updates_both_adapters_and_closes_existing_http_tunnel() { +async fn policy_reload_updates_transparent_requests_and_closes_existing_http_stream() { let server = KeepAliveHttpServer::start() .await .expect("start keep-alive HTTP server"); @@ -950,7 +919,7 @@ async fn policy_reload_updates_both_adapters_and_closes_existing_http_tunnel() { .await .expect("wait for policy A"); - let persistent_script = persistent_connect_script(TEST_SERVER_HOST, server.port); + let persistent_script = persistent_transparent_script(TEST_SERVER_HOST, server.port); guard .exec(&[ "sh", @@ -960,7 +929,7 @@ async fn policy_reload_updates_both_adapters_and_closes_existing_http_tunnel() { &persistent_script, ]) .await - .expect("start persistent CONNECT client"); + .expect("start persistent transparent client"); wait_for_sandbox_file( &guard, "/tmp/proxy-reload-ready", @@ -968,16 +937,19 @@ async fn policy_reload_updates_both_adapters_and_closes_existing_http_tunnel() { ) .await; - let status_script = proxy_status_script(TEST_SERVER_HOST, server.port); + let status_script = transparent_status_script(TEST_SERVER_HOST, server.port); let before = guard .exec(&["python3", "-c", &status_script]) .await - .expect("exercise both adapters before reload"); + .expect("exercise transparent requests before reload"); let before = parse_json_line(&before); - assert_eq!(before["connect"], 200, "CONNECT before reload: {before}"); assert_eq!( - before["forward"], 200, - "forward HTTP before reload: {before}" + before["first"], 200, + "first request before reload: {before}" + ); + assert_eq!( + before["second"], 200, + "second request before reload: {before}" ); run_cli(&[ @@ -996,7 +968,7 @@ async fn policy_reload_updates_both_adapters_and_closes_existing_http_tunnel() { guard .exec(&["sh", "-c", "touch /tmp/proxy-reload-go"]) .await - .expect("release persistent CONNECT client"); + .expect("release persistent transparent client"); let stale_tunnel = wait_for_sandbox_file( &guard, "/tmp/proxy-reload-result", @@ -1006,16 +978,16 @@ async fn policy_reload_updates_both_adapters_and_closes_existing_http_tunnel() { let stale_tunnel = parse_json_line(&stale_tunnel); assert_eq!( stale_tunnel["failed_closed"], true, - "existing CONNECT HTTP stream forwarded after policy reload: {stale_tunnel}" + "existing transparent HTTP stream forwarded after policy reload: {stale_tunnel}" ); let after = guard .exec(&["python3", "-c", &status_script]) .await - .expect("exercise both adapters after reload"); + .expect("exercise transparent requests after reload"); let after = parse_json_line(&after); - assert_eq!(after["connect"], 403, "CONNECT after reload: {after}"); - assert_eq!(after["forward"], 403, "forward HTTP after reload: {after}"); + assert_ne!(after["first"], 200, "first request after reload: {after}"); + assert_ne!(after["second"], 200, "second request after reload: {after}"); guard.cleanup().await; } @@ -1052,14 +1024,20 @@ async fn ambiguous_policy_update_is_rejected_without_replacing_active_policy() { .await .expect("wait for valid policy"); - let status_script = proxy_status_script(TEST_SERVER_HOST, server.port); + let status_script = transparent_status_script(TEST_SERVER_HOST, server.port); let before = guard .exec(&["python3", "-c", &status_script]) .await - .expect("exercise both adapters before invalid update"); + .expect("exercise transparent requests before invalid update"); let before = parse_json_line(&before); - assert_eq!(before["connect"], 200, "CONNECT before update: {before}"); - assert_eq!(before["forward"], 200, "forward before update: {before}"); + assert_eq!( + before["first"], 200, + "first request before update: {before}" + ); + assert_eq!( + before["second"], 200, + "second request before update: {before}" + ); let history_before = run_cli(&["policy", "list", &guard.name]) .await .expect("list policy history before rejected update"); @@ -1093,15 +1071,15 @@ async fn ambiguous_policy_update_is_rejected_without_replacing_active_policy() { let after_rejection = guard .exec(&["python3", "-c", &status_script]) .await - .expect("exercise both adapters after rejected update"); + .expect("exercise transparent requests after rejected update"); let after_rejection = parse_json_line(&after_rejection); assert_eq!( - after_rejection["connect"], 200, - "CONNECT should keep using the active valid policy: {after_rejection}" + after_rejection["first"], 200, + "first request should keep using the active valid policy: {after_rejection}" ); assert_eq!( - after_rejection["forward"], 200, - "forward HTTP should keep using the active valid policy: {after_rejection}" + after_rejection["second"], 200, + "second request should keep using the active valid policy: {after_rejection}" ); assert!( server.connection_count() > connections_before_rejection, @@ -1112,69 +1090,25 @@ async fn ambiguous_policy_update_is_rejected_without_replacing_active_policy() { } #[tokio::test] -async fn destination_denial_modes_match_across_connect_and_forward_adapters() { +async fn transparent_destination_denials_fail_connect_with_eacces() { let policy = write_destination_denial_policy().expect("write destination denial policy"); let policy_path = policy_path(&policy); let script = r#" import json -import os import socket -import urllib.parse - -proxy_url = next( - os.environ[name] - for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - if os.environ.get(name) -) -parsed = urllib.parse.urlparse(proxy_url) - -def read_response(sock): - data = b"" - while b"\r\n\r\n" not in data: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - headers, _, body = data.partition(b"\r\n\r\n") - length = 0 - for line in headers.split(b"\r\n")[1:]: - if line.lower().startswith(b"content-length:"): - length = int(line.split(b":", 1)[1].strip()) - while len(body) < length: - chunk = sock.recv(4096) - if not chunk: - break - body += chunk - status = int(headers.split(None, 2)[1]) - return {"status": status, "body": json.loads(body.decode())} - -def connect_result(host, port): - with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: - target = f"{host}:{port}" - sock.sendall(f"CONNECT {target} HTTP/1.1\r\nHost: {target}\r\n\r\n".encode()) - return read_response(sock) - -def forward_result(host, port): - with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: - target = f"{host}:{port}" - sock.sendall( - f"GET http://{target}/probe HTTP/1.1\r\n" - f"Host: {target}\r\nConnection: close\r\n\r\n".encode() - ) - return read_response(sock) targets = { "metadata": ("169.254.169.254", 80), - "loopback": ("127.0.0.1", 80), "control_plane": ("203.0.113.10", 6443), "outside_allowed_ips": ("203.0.113.10", 8080), } result = {} for name, target in targets.items(): - result[name] = { - "connect": connect_result(*target), - "forward": forward_result(*target), - } + try: + with socket.create_connection(target, timeout=10): + result[name] = 0 + except OSError as error: + result[name] = error.errno print(json.dumps(result, sort_keys=True)) "#; @@ -1182,81 +1116,46 @@ print(json.dumps(result, sort_keys=True)) .await .expect("sandbox create"); let result = parse_json_line(&guard.create_output); - for name in [ - "metadata", - "loopback", - "control_plane", - "outside_allowed_ips", - ] { - for adapter in ["connect", "forward"] { - assert_eq!( - result[name][adapter]["status"], 403, - "{name} {adapter}: {result}" - ); - assert_eq!( - result[name][adapter]["body"]["error"], "ssrf_denied", - "{name} {adapter}: {result}" - ); - } + for name in ["metadata", "control_plane", "outside_allowed_ips"] { + assert_eq!(result[name], 13, "{name} should fail with EACCES: {result}"); } - assert_eq!( - result["metadata"]["connect"]["body"]["detail"], - "CONNECT 169.254.169.254:80 blocked: declared endpoint check failed" - ); - assert_eq!( - result["metadata"]["forward"]["body"]["detail"], - "GET 169.254.169.254:80 blocked: declared endpoint check failed" - ); - assert_eq!( - result["control_plane"]["connect"]["body"]["detail"], - "CONNECT 203.0.113.10:6443 blocked: allowed_ips check failed" - ); - assert_eq!( - result["outside_allowed_ips"]["forward"]["body"]["detail"], - "GET 203.0.113.10:8080 blocked: allowed_ips check failed" - ); } #[tokio::test] -async fn explicit_allowed_ips_and_implicit_ip_literals_succeed_through_both_adapters() { - let resolver = SandboxGuard::create(&[ - "--", - "python3", - "-c", - "import socket; print('GATEWAY_IP=' + socket.gethostbyname('host.openshell.internal'))", - ]) - .await - .expect("resolve host gateway inside sandbox"); - let gateway_ip = resolver - .create_output - .lines() - .find_map(|line| line.trim().strip_prefix("GATEWAY_IP=")) - .expect("sandbox gateway IPv4 output") - .parse::() - .expect("host gateway must resolve to IPv4 for this e2e"); - - // Rootless Podman with pasta exposes its trusted host-gateway alias as a - // link-local address. The hostname receives a narrow runtime exemption, - // but the equivalent raw IP literal must remain hard-blocked. Other - // drivers still exercise the successful IP-literal path below. - if gateway_ip.is_loopback() || gateway_ip.is_link_local() || gateway_ip.is_unspecified() { - eprintln!( - "skipping IP-literal success assertions: host gateway {gateway_ip} is always blocked" - ); +async fn explicit_allowed_ips_and_implicit_ip_literals_succeed_transparently() { + if e2e_network_name().is_none() { + eprintln!("skipping IP-literal success assertions without a shared container network"); return; } - let gateway_ip = gateway_ip.to_string(); + const HTTP_SERVER: &str = r#" +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer - let explicit_server = KeepAliveHttpServer::start() - .await - .expect("start explicit allowed_ips server"); - let implicit_server = KeepAliveHttpServer::start() - .await - .expect("start implicit IP-literal server"); - let policy = - write_ip_literal_success_policy(&gateway_ip, explicit_server.port, implicit_server.port) - .expect("write IP literal policy"); +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = b"ok" + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + pass + +ThreadingHTTPServer(("0.0.0.0", 8000), Handler).serve_forever() +"#; + let explicit_server = + SupportContainer::start_python("explicit-ip.openshell.test", HTTP_SERVER, 8000) + .await + .expect("start explicit allowed_ips support container"); + let implicit_server = + SupportContainer::start_python("implicit-ip.openshell.test", HTTP_SERVER, 8000) + .await + .expect("start implicit IP-literal support container"); + let explicit_ip = explicit_server.ip().expect("explicit support container IP"); + let implicit_ip = implicit_server.ip().expect("implicit support container IP"); + let policy = write_ip_literal_success_policy(&explicit_ip, &implicit_ip, 8000) + .expect("write IP literal policy"); let policy_path = policy_path(&policy); let mut guard = SandboxGuard::create_keep_with_args( &["--policy", &policy_path], @@ -1266,17 +1165,21 @@ async fn explicit_allowed_ips_and_implicit_ip_literals_succeed_through_both_adap .await .expect("create keep sandbox"); - for (mode, port) in [ - ("explicit_allowed_ips", explicit_server.port), - ("implicit_ip_literal", implicit_server.port), + for (mode, destination) in [ + ("explicit_allowed_ips", explicit_ip.as_str()), + ("implicit_ip_literal", implicit_ip.as_str()), ] { let output = guard - .exec(&["python3", "-c", &proxy_status_script(&gateway_ip, port)]) + .exec(&[ + "python3", + "-c", + &transparent_status_script(destination, 8000), + ]) .await .unwrap_or_else(|error| panic!("exercise {mode}: {error}")); let statuses = parse_json_line(&output); - assert_eq!(statuses["connect"], 200, "{mode} CONNECT: {statuses}"); - assert_eq!(statuses["forward"], 200, "{mode} forward: {statuses}"); + assert_eq!(statuses["first"], 200, "{mode} first request: {statuses}"); + assert_eq!(statuses["second"], 200, "{mode} second request: {statuses}"); } guard.cleanup().await; @@ -1290,28 +1193,13 @@ async fn tls_skip_connect_relays_opaque_bytes_bidirectionally() { let policy_path = policy_path(&policy); let script = format!( r#" -import os import socket -import urllib.parse HOST = {host:?} PORT = {port} PAYLOAD = bytes([0x00, 0xff, 0x13, 0x37, 0x80, 0x0a]) + b"not-http-or-tls" + bytes(range(64)) -proxy_url = next( - os.environ[name] - for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - if os.environ.get(name) -) -parsed = urllib.parse.urlparse(proxy_url) -with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: - target = f"{{HOST}}:{{PORT}}" - sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) - response = b"" - while b"\r\n\r\n" not in response: - response += sock.recv(4096) - if int(response.split(None, 2)[1]) != 200: - raise RuntimeError("CONNECT was denied") +with socket.create_connection((HOST, PORT), timeout=10) as sock: sock.sendall(PAYLOAD) echoed = b"" while len(echoed) < len(PAYLOAD): @@ -1338,7 +1226,7 @@ print("RAW_RELAY_OK") } #[tokio::test] -async fn middleware_redacts_request_bodies_through_both_adapters() { +async fn middleware_redacts_transparent_request_bodies() { let server = RequestBodyEchoServer::start() .await .expect("start request body echo server"); @@ -1348,21 +1236,12 @@ async fn middleware_redacts_request_bodies_through_both_adapters() { let script = format!( r#" import json -import os import socket -import urllib.parse HOST = {host:?} PORT = {port} SECRET = "sk-1234567890abcdef" -proxy_url = next( - os.environ[name] - for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - if os.environ.get(name) -) -parsed = urllib.parse.urlparse(proxy_url) - def read_response(sock): data = b"" while b"\r\n\r\n" not in data: @@ -1395,22 +1274,12 @@ def request_bytes(target): "Connection: close\r\n\r\n" ).encode() + body -target = f"{{HOST}}:{{PORT}}" -with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as forward_sock: - forward_sock.sendall(request_bytes(f"http://{{target}}/middleware")) - forward = read_response(forward_sock) - -with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as connect_sock: - connect_sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) - connect_response = b"" - while b"\r\n\r\n" not in connect_response: - connect_response += connect_sock.recv(4096) - if int(connect_response.split(None, 2)[1]) != 200: - raise RuntimeError("CONNECT was denied") - connect_sock.sendall(request_bytes("/middleware")) - connect = read_response(connect_sock) - -print(json.dumps({{"connect": connect, "forward": forward}}, sort_keys=True)) +def request_once(): + with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall(request_bytes("/middleware")) + return read_response(sock) + +print(json.dumps({{"first": request_once(), "second": request_once()}}, sort_keys=True)) "#, host = TEST_SERVER_HOST, port = server.port, @@ -1420,44 +1289,29 @@ print(json.dumps({{"connect": connect, "forward": forward}}, sort_keys=True)) .await .expect("sandbox create"); let result = parse_json_line(&guard.create_output); - for adapter in ["connect", "forward"] { + for request in ["first", "second"] { assert_eq!( - result[adapter]["api_key"], "[REDACTED]", - "{adapter} did not deliver the middleware-transformed body: {result}" + result[request]["api_key"], "[REDACTED]", + "{request} did not deliver the middleware-transformed body: {result}" ); } } #[tokio::test] -async fn fail_closed_middleware_blocks_uninspectable_connect_payload_before_upstream() { +async fn fail_closed_middleware_blocks_uninspectable_transparent_payload_before_upstream() { let server = EchoServer::start().await.expect("start TCP echo server"); let policy = write_middleware_policy(TEST_SERVER_HOST, server.port, "", "fail_closed") .expect("write fail-closed middleware policy"); let policy_path = policy_path(&policy); let script = format!( r#" -import os import socket -import urllib.parse HOST = {host:?} PORT = {port} PAYLOAD = bytes([0x00, 0xff, 0x13, 0x37]) + b"not-http-or-tls" -proxy_url = next( - os.environ[name] - for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - if os.environ.get(name) -) -parsed = urllib.parse.urlparse(proxy_url) -with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: - target = f"{{HOST}}:{{PORT}}" - sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) - response = b"" - while b"\r\n\r\n" not in response: - response += sock.recv(4096) - if int(response.split(None, 2)[1]) != 200: - raise RuntimeError("CONNECT was denied before tunnel establishment") +with socket.create_connection((HOST, PORT), timeout=10) as sock: sock.sendall(PAYLOAD) denial = b"" while True: @@ -1518,7 +1372,7 @@ print("UNINSPECTABLE_MIDDLEWARE_BLOCKED") } #[tokio::test] -async fn fail_open_middleware_bypasses_uninspectable_tls_skip_connect() { +async fn fail_open_middleware_bypasses_uninspectable_transparent_tls_skip() { let server = EchoServer::start().await.expect("start TCP echo server"); let policy = write_middleware_policy( TEST_SERVER_HOST, @@ -1530,28 +1384,13 @@ async fn fail_open_middleware_bypasses_uninspectable_tls_skip_connect() { let policy_path = policy_path(&policy); let script = format!( r#" -import os import socket -import urllib.parse HOST = {host:?} PORT = {port} PAYLOAD = bytes([0x00, 0xff, 0x13, 0x37, 0x80]) + b"middleware-bypass" -proxy_url = next( - os.environ[name] - for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - if os.environ.get(name) -) -parsed = urllib.parse.urlparse(proxy_url) -with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: - target = f"{{HOST}}:{{PORT}}" - sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) - response = b"" - while b"\r\n\r\n" not in response: - response += sock.recv(4096) - if int(response.split(None, 2)[1]) != 200: - raise RuntimeError("CONNECT was denied") +with socket.create_connection((HOST, PORT), timeout=10) as sock: sock.sendall(PAYLOAD) echoed = b"" while len(echoed) < len(PAYLOAD): @@ -1588,7 +1427,7 @@ print("UNINSPECTABLE_MIDDLEWARE_BYPASSED") } #[tokio::test] -async fn forward_pipeline_never_reaches_upstream_as_first_request_overflow() { +async fn transparent_pipeline_never_reaches_upstream_as_first_request_overflow() { let server = PipelineProbeServer::start() .await .expect("start pipeline probe server"); @@ -1603,26 +1442,18 @@ async fn forward_pipeline_never_reaches_upstream_as_first_request_overflow() { let policy_path = policy_path(&policy); let script = format!( r#" -import os import socket -import urllib.parse -proxy_url = next( - os.environ[name] - for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - if os.environ.get(name) -) -parsed = urllib.parse.urlparse(proxy_url) target = "{host}:{port}" first = ( - f"GET http://{{target}}/allowed HTTP/1.1\r\n" + f"GET /allowed HTTP/1.1\r\n" f"Host: {{target}}\r\nConnection: keep-alive\r\n\r\n" ) second = ( - f"POST http://{{target}}/blocked HTTP/1.1\r\n" + f"POST /blocked HTTP/1.1\r\n" f"Host: {{target}}\r\nContent-Length: 0\r\n\r\n" ) -with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: +with socket.create_connection(({host:?}, {port}), timeout=10) as sock: sock.sendall((first + second).encode()) response = b"" while True: @@ -1630,9 +1461,11 @@ with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) if not chunk: break response += chunk -if response.count(b"HTTP/1.1 ") != 1 or b" 200 " not in response.split(b"\r\n", 1)[0]: +responses = response.count(b"HTTP/1.1 ") +first_status = response.split(b"\r\n", 1)[0] +if responses != 2 or b" 200 " not in first_status or b"HTTP/1.1 403 Forbidden" not in response: raise RuntimeError(f"unexpected pipelined response: {{response!r}}") -print("FORWARD_PIPELINE_CLOSED") +print("TRANSPARENT_PIPELINE_DENIED") "#, host = TEST_SERVER_HOST, port = server.port, @@ -1642,8 +1475,8 @@ print("FORWARD_PIPELINE_CLOSED") .await .expect("sandbox create"); assert!( - guard.create_output.contains("FORWARD_PIPELINE_CLOSED"), - "forward proxy did not close after one response:\n{}", + guard.create_output.contains("TRANSPARENT_PIPELINE_DENIED"), + "transparent HTTP stream did not deny the disallowed pipelined request:\n{}", guard.create_output ); @@ -1657,7 +1490,7 @@ print("FORWARD_PIPELINE_CLOSED") } #[tokio::test] -async fn http_credentials_are_rewritten_in_headers_and_bodies_for_both_adapters() { +async fn http_credentials_are_rewritten_in_transparent_headers_and_bodies() { let _provider_lock = PROVIDER_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); @@ -1682,21 +1515,11 @@ async fn http_credentials_are_rewritten_in_headers_and_bodies_for_both_adapters( import json import os import socket -import urllib.parse HOST = {host:?} PORT = {port} TOKEN = os.environ[{token_env:?}] -def proxy_parts(): - proxy_url = next( - os.environ[name] - for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - if os.environ.get(name) - ) - parsed = urllib.parse.urlparse(proxy_url) - return parsed.hostname, parsed.port or 80 - def read_response(sock): data = b"" while b"\r\n\r\n" not in data: @@ -1730,23 +1553,12 @@ def request_bytes(target): "Connection: close\r\n\r\n" ).encode() + body -proxy_host, proxy_port = proxy_parts() -target = f"{{HOST}}:{{PORT}}" -with socket.create_connection((proxy_host, proxy_port), timeout=10) as forward_sock: - forward_sock.sendall(request_bytes(f"http://{{target}}/probe")) - forward = read_response(forward_sock) - -with socket.create_connection((proxy_host, proxy_port), timeout=10) as connect_sock: - connect_sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) - connect_response = b"" - while b"\r\n\r\n" not in connect_response: - connect_response += connect_sock.recv(4096) - if int(connect_response.split(None, 2)[1]) != 200: - raise RuntimeError("CONNECT was denied") - connect_sock.sendall(request_bytes("/probe")) - connect = read_response(connect_sock) - -print(json.dumps({{"connect": connect, "forward": forward}}, sort_keys=True)) +def request_once(): + with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall(request_bytes("/probe")) + return read_response(sock) + +print(json.dumps({{"first": request_once(), "second": request_once()}}, sort_keys=True)) "#, host = TEST_SERVER_HOST, port = server.port, @@ -1772,18 +1584,18 @@ print(json.dumps({{"connect": connect, "forward": forward}}, sort_keys=True)) let guard = result.expect("sandbox create"); let result = parse_json_line(&guard.create_output); - for adapter in ["connect", "forward"] { + for request in ["first", "second"] { assert_eq!( - result[adapter]["header_resolved"], true, - "{adapter} header placeholder was not resolved: {result}" + result[request]["header_resolved"], true, + "{request} header placeholder was not resolved: {result}" ); assert_eq!( - result[adapter]["body_resolved"], true, - "{adapter} body placeholder was not resolved: {result}" + result[request]["body_resolved"], true, + "{request} body placeholder was not resolved: {result}" ); assert_eq!( - result[adapter]["saw_placeholder"], false, - "{adapter} leaked an unresolved placeholder upstream: {result}" + result[request]["saw_placeholder"], false, + "{request} leaked an unresolved placeholder upstream: {result}" ); } assert!( diff --git a/e2e/rust/tests/transparent_tcp.rs b/e2e/rust/tests/transparent_tcp.rs index 1027654641..6f3f751193 100644 --- a/e2e/rust/tests/transparent_tcp.rs +++ b/e2e/rust/tests/transparent_tcp.rs @@ -356,7 +356,7 @@ print('transparent-tcp-e2e-ok') let logs = wait_for_sandbox_logs(&sandbox.name, |logs| { logs.contains(&format!("-> {FIXTURE_ALIAS}:{FIXTURE_PORT}")) - && logs.contains("transparent_tcp_port_mismatch") + && logs.contains("Denied staged transparent connection") }) .await .expect("wait for sandbox logs"); @@ -364,7 +364,10 @@ print('transparent-tcp-e2e-ok') logs.contains(&format!("-> {FIXTURE_ALIAS}:{FIXTURE_PORT}")), "{logs}" ); - assert!(logs.contains("transparent_tcp_port_mismatch"), "{logs}"); + assert!( + logs.contains("Denied staged transparent connection"), + "{logs}" + ); sandbox.cleanup().await; } diff --git a/rfc/0003-gateway-configuration/README.md b/rfc/0003-gateway-configuration/README.md index 8007236d7a..981a120b93 100644 --- a/rfc/0003-gateway-configuration/README.md +++ b/rfc/0003-gateway-configuration/README.md @@ -8,16 +8,16 @@ state: implemented ## Summary -Introduce a TOML-based configuration file for the OpenShell gateway that unifies gateway settings — core server options, TLS, OIDC, observability listeners, and per-driver parameters — under a single structured file. CLI flags and supported `OPENSHELL_*` environment variables retain higher precedence. Schema version 2 intentionally rejects legacy file fields and locations. +Introduce a TOML-based configuration file for the OpenShell gateway that unifies all gateway settings — core server options, TLS, OIDC, observability listeners, and per-driver parameters — under a single structured file, while preserving full backwards compatibility with the existing CLI flags and `OPENSHELL_*` environment variables. ## Motivation -Before this RFC, the gateway was configured exclusively through CLI flags and `OPENSHELL_*` environment variables. This worked for simple single-node deployments but broke down as deployments grew: +The gateway today is configured exclusively through CLI flags and `OPENSHELL_*` environment variables. This works for simple single-node deployments but breaks down as deployments grow: -- **Too many flags** — the gateway exposed roughly 40 configurable parameters (TLS, OIDC, four compute drivers, three listeners). Long `docker run` commands and `args:` arrays in Kubernetes manifests were hard to read, diff, and audit. -- **Driver coupling** — Docker, Podman, Kubernetes, and VM drivers shared one flat CLI namespace with no structural separation. Most flags applied to only one driver, but CLI syntax did not express that ownership. -- **Helm friction** — The chart's `statefulset.yaml` carried a long `env:` block of `OPENSHELL_*` variables that each mapped to a `values.yaml` key. A mounted configuration file reduces the chart's templating surface. -- **Secrets management** — Environment-only configuration did not compose naturally with Kubernetes `ConfigMap` and projected `Secret` volumes. +- **Too many flags** — the gateway has ~40 configurable parameters today (TLS, OIDC, four compute drivers, three listeners). Long `docker run` commands and `args:` arrays in Kubernetes manifests are hard to read, diff, and audit. +- **Driver coupling** — Docker, Podman, Kubernetes, and VM drivers all live in the same flat CLI namespace, with no structural separation. Most flags only apply to one driver, but there is no way to express that in CLI form. +- **Helm friction** — The chart's `statefulset.yaml` already carries a long `env:` block of `OPENSHELL_*` variables that each map to a `values.yaml` key. A config file can be mounted as a single `ConfigMap` and reduces the chart's templating surface significantly. +- **Secrets management** — Injecting secrets (TLS material paths, database URL, OIDC settings) via environment variables is functional but not idiomatic for Kubernetes. A file-based format opens the door to projected secrets and volume mounts that compose cleanly with the non-secret config. ## Non-goals @@ -48,7 +48,7 @@ The file path is provided via: OPENSHELL_GATEWAY_CONFIG=/path/to/gateway.toml ``` -The file must have a `.toml` extension. A missing path is a hard error. A configured file must declare the exact supported schema version; an empty existing file is rejected. +The file must have a `.toml` extension. A missing path is a hard error; an empty existing file is treated as "no configuration" — the gateway falls back to defaults and to whatever the CLI/env supply. ### TOML schema @@ -58,7 +58,7 @@ The file is rooted at an `[openshell]` table. This namespacing reserves room for ```toml [openshell] -version = 2 # required schema version +version = 1 # optional; reserved for future schema migrations # ────────────────────────────────────────────────────────────────────────────── # Gateway-wide settings @@ -72,9 +72,10 @@ metrics_bind_address = "0.0.0.0:9090" # optional; omit to disable # Logging log_level = "info" -# Compute driver — exactly one driver may be active. When omitted, the gateway -# auto-detects a driver (kubernetes → podman → docker). VM is never auto-detected. -compute_driver = "kubernetes" +# Compute drivers — list of driver names whose [openshell.drivers.] +# tables should be activated. When empty, the gateway auto-detects a driver +# (kubernetes → podman → docker). VM is never auto-detected. +compute_drivers = ["kubernetes"] # Note: database_url is a secret and must be supplied via OPENSHELL_DB_URL # (or --db-url) — it is NOT permitted in the file. @@ -87,17 +88,12 @@ server_sans = ["openshell", "*.dev.openshell.localhost"] enable_loopback_service_http = true # ────────────────────────────────────────────────────────────────────────────── -# TLS / mTLS — package-managed local TLS may supply listener defaults. +# TLS / mTLS — when omitted, the gateway listens plaintext (sets --disable-tls) # ────────────────────────────────────────────────────────────────────────────── -# Mirrors --disable-tls / OPENSHELL_DISABLE_TLS. Set true explicitly for a -# plaintext listener; guest TLS fields must then be omitted. +# Mirrors --disable-tls / OPENSHELL_DISABLE_TLS. When true, the gateway +# ignores the [openshell.gateway.tls] table below. disable_tls = false -# Gateway-owned TLS bundle injected into the selected local driver. -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" - [openshell.gateway.tls] cert_path = "/etc/openshell/certs/gateway.pem" key_path = "/etc/openshell/certs/gateway-key.pem" @@ -117,15 +113,15 @@ scopes_claim = "" # empty disables scope enforcement # ────────────────────────────────────────────────────────────────────────────── # Compute drivers — each table is owned and parsed by its driver crate. -# Only the selected or auto-detected driver's table is activated. +# Only tables for drivers listed in compute_drivers are activated. # ────────────────────────────────────────────────────────────────────────────── [openshell.drivers.kubernetes] namespace = "openshell" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "if_not_present" +image_pull_policy = "IfNotPresent" supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" -supervisor_image_pull_policy = "if_not_present" +supervisor_image_pull_policy = "IfNotPresent" grpc_endpoint = "https://host.openshell.internal:8080" client_tls_secret_name = "openshell-sandbox-tls" host_gateway_ip = "10.0.0.1" @@ -133,20 +129,25 @@ ssh_socket_path = "/run/openshell/ssh.sock" [openshell.drivers.docker] default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "if_not_present" -sandbox_label = "docker-dev" +image_pull_policy = "IfNotPresent" +sandbox_namespace = "docker-dev" grpc_endpoint = "https://host.openshell.internal:8080" network_name = "openshell" -supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" # optional override -supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" # used to extract bin +supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" # contains sandbox + supervisor +guest_tls_ca = "/etc/openshell/certs/ca.pem" +guest_tls_cert = "/etc/openshell/certs/client.pem" +guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.podman] socket_path = "/run/podman/podman.sock" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "if_not_present" # always | if_not_present | never | newer +image_pull_policy = "missing" # Podman vocabulary: always | missing | never | newer supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" network_name = "openshell" stop_timeout_secs = 10 +guest_tls_ca = "/etc/openshell/certs/ca.pem" +guest_tls_cert = "/etc/openshell/certs/client.pem" +guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.vm] state_dir = "/var/lib/openshell/vm" @@ -155,6 +156,9 @@ grpc_endpoint = "https://host.containers.internal:8080" vcpus = 2 mem_mib = 2048 krun_log_level = 1 +guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" +guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" +guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" ``` ### Driver configuration @@ -162,12 +166,12 @@ krun_log_level = 1 Each `[openshell.drivers.]` table is extracted from the parsed file and handed to the driver's initialization function as a raw TOML value. The driver is then responsible for: 1. **Parsing** — deserializing the table into its own typed config struct (e.g. `KubernetesComputeConfig`, `DockerComputeConfig`, `PodmanComputeConfig`, `VmComputeConfig`). -2. **Validation** — applying cross-field checks specific to that driver. Gateway-owned guest TLS paths are validated as one bundle and injected only into the selected local driver before this step. +2. **Validation** — applying cross-field checks specific to that driver (e.g. requiring TLS triplets when sandbox-side mTLS is enabled). 3. **Consumption** — using the resulting struct to initialize internal state. Driver authors define and own their config schema. Adding a new driver does not require changes to the gateway's core `Config` struct or to this RFC. -`[openshell.drivers.]` tables for drivers other than the selected or auto-detected driver are parsed for syntax but not activated. +`[openshell.drivers.]` tables for drivers not listed in `compute_drivers` (and not the auto-detected driver) are parsed for syntax but not activated. ### Merge semantics @@ -202,26 +206,25 @@ Deserialization uses `#[serde(deny_unknown_fields)]` at every table level. An un The following cross-field validations are applied after merging file + env + CLI: - `bind_address`, `health_bind_address`, and `metrics_bind_address` must all use distinct ports when set. -- Gateway listener TLS requires `cert_path` and `key_path`; `client_ca_path` is required only for listener client-certificate verification. TLS-enabled Docker, Podman, and VM drivers also require a complete gateway-owned guest CA, certificate, and key bundle. Kubernetes projects guest TLS through a Secret instead. +- When `[openshell.gateway.tls]` is present, all three of `cert_path`, `key_path`, and `client_ca_path` must be present (either from the file or from CLI/env). Partial TLS configuration is an error. - `database_url` must be non-empty after merging env + CLI — every supported driver requires it. The field is not accepted from the file (see Secrets above). -- `compute_driver` selects exactly one driver. When omitted, the gateway falls back to auto-detection. A custom driver requires a named table with `socket_path`, unless startup supplies an explicit socket override. The legacy `compute_drivers` list is rejected. +- `compute_drivers` may be empty; in that case the gateway falls back to auto-detection. If the list contains a driver name with no matching `[openshell.drivers.]` table, the driver runs with its built-in defaults. -### Schema compatibility +### Backwards compatibility -Schema version 2 requires `version = 2`, a singular `compute_driver` when a driver is selected, and driver-owned fields under `[openshell.drivers.]`. Legacy schema versions and `compute_drivers` lists are rejected. `OPENSHELL_DB_URL` remains a required process input and is not accepted from the file. +The existing CLI interface is fully preserved. All flags continue to work exactly as before. The `--config` flag is new and additive. `OPENSHELL_DB_URL` remains a required process input (it is not accepted from the file). ### Example: minimal Kubernetes deployment ```toml [openshell] -version = 2 +version = 1 [openshell.gateway] -bind_address = "0.0.0.0:8080" -compute_driver = "kubernetes" +bind_address = "0.0.0.0:8080" +compute_drivers = ["kubernetes"] # database_url comes from env (e.g. valueFrom.secretKeyRef). -# The gateway runs plaintext behind Envoy / ingress. -disable_tls = true +# No [openshell.gateway.tls] → plaintext listener (gateway runs behind Envoy / ingress). [openshell.drivers.kubernetes] namespace = "agents" @@ -232,7 +235,12 @@ grpc_endpoint = "https://openshell-gateway.agents.svc:8080" ### Helm integration -The Helm chart renders schema-v2 gateway TOML into a `ConfigMap`, mounts it at `/etc/openshell/gateway.toml`, and starts the gateway with that file. Secret process inputs such as `OPENSHELL_DB_URL` remain `Secret`-backed environment entries and retain higher precedence. Kubernetes projects sandbox guest TLS through its configured Secret rather than placing host guest-certificate paths in the gateway TOML. +The Helm chart today renders a long `env:` block in `templates/statefulset.yaml`, with each `OPENSHELL_*` variable mapped to a `values.yaml` key. This RFC's adoption replaces that block with: + +1. A new `gateway.config` value tree (TOML-shaped YAML) in `values.yaml`. +2. A new `ConfigMap` template that renders the values into a TOML document via Helm's `tpl`. +3. A volume mount of the `ConfigMap` at `/etc/openshell/gateway.toml` and a `--config` flag in the gateway container's `args`. +4. Continued use of a `Secret`-backed `env:` entry for `OPENSHELL_DB_URL` (which never lives in the `ConfigMap`), plus optional projections for TLS material paths. The CLI/env precedence above means any `Secret`-backed env var also wins over a value in the `ConfigMap`. ```yaml # values.yaml excerpt @@ -241,7 +249,7 @@ gateway: bind_address: "0.0.0.0:8080" health_bind_address: "0.0.0.0:8081" metrics_bind_address: "0.0.0.0:9090" - compute_driver: "kubernetes" + compute_drivers: ["kubernetes"] drivers: kubernetes: namespace: agents @@ -251,15 +259,23 @@ gateway: The chart owners can migrate one section at a time: `OPENSHELL_*` env vars and the `ConfigMap` coexist during the transition, with env continuing to override the file. -## Implementation +## Implementation plan + +No part of this RFC has shipped yet. The work breaks down as: -The implemented gateway loader parses TOML with `serde`, merges file values below environment and CLI sources, and rejects unknown fields. Each compute driver deserializes only its named table. Helm renders schema-v2 TOML into a ConfigMap, while secret process inputs remain environment-backed. Package templates, examples, tests, and the gateway architecture documentation use the same canonical schema. +1. **Add a config-file loader to `openshell-server`** — define a `GatewayConfigFile` struct that mirrors the schema above, parse it with `serde` + `toml`, and merge it into `openshell_core::Config` plus the per-driver structs in `compute/`. +2. **Wire the merge into `cli.rs`** — add `--config` / `OPENSHELL_GATEWAY_CONFIG`, gate each existing flag's "apply from file" path on clap `ValueSource::DefaultValue`, and run cross-field validation after the merge. +3. **Per-driver deserialization** — give each driver crate (`openshell-driver-{kubernetes,docker,podman,vm}`) a `from_toml` (or `serde::Deserialize`) entry point so the gateway can hand each driver its own table. +4. **Test coverage** — file parsing, env-overrides-file, CLI-overrides-env, partial TLS error, port-collision error, unknown-field rejection, missing driver table fallback. +5. **Helm chart migration** — add `gateway.config` value tree, render the `ConfigMap`, mount it, switch the gateway container to `--config`. Keep the `OPENSHELL_*` env names available as opt-in overrides for secrets. +6. **Example file** — ship the per-driver examples on the published docs reference at `docs/reference/gateway-config.mdx`. +7. **Architecture doc update** — reflect the new config sources and precedence in `architecture/gateway.md`. ## Risks -- **Serde `deny_unknown_fields` is strict** — any field name change in `openshell_core::Config` or in a driver's config struct becomes a breaking change for anyone using the file. Treat field renames as versioned schema changes and surface migration errors clearly. +- **Serde `deny_unknown_fields` is strict** — any field name change in `openshell_core::Config` or in a driver's config struct becomes a breaking change for anyone using the file. Mitigate by treating field renames as breaking, keeping the `version` field reserved for schema migrations, and surfacing rename errors clearly. - **Secrets in the file** — `database_url` is excluded from the schema entirely (env / CLI only). OIDC settings remain allowed in the file because none of them are credentials in isolation. Operators should still prefer env-var injection for any field that would live in a `Secret` rather than a `ConfigMap` (TLS material paths, restricted-environment OIDC issuers, etc.). Documentation must call this out prominently. -- **Partial TLS configuration** — listener and guest TLS are separate complete-bundle contracts. Startup rejects partial bundles and identifies the missing configuration before constructing a driver. +- **Partial TLS configuration** — the hard error on partial TLS config is the right UX, but the error message must clearly identify which source (file vs. CLI/env) is missing which field, since the file's `[openshell.gateway.tls]` table is all-or-nothing while the CLI flags are independent. - **Driver schema drift** — once each driver owns its own TOML table, driver releases can change field names independently of the gateway. The gateway's `version` field does not protect against driver-side breakage; document driver-config stability separately. ## Alternatives @@ -278,7 +294,8 @@ The implemented gateway loader parses TOML with `serde`, merges file values belo ## Open questions -1. **Directory-based config (`conf.d` pattern)** — a `--config-dir` flag that globs all `*.toml` files in a directory, sorts them alphabetically, and deep-merges them in order (later files win per key). CLI/env overrides still sit above everything. This maps cleanly to Kubernetes: a base `ConfigMap` as `10-base.toml`, driver config as `20-kubernetes.toml`, and credentials from a projected `Secret` as `90-credentials.toml` — all mounted into the same directory without a monolithic file. This is the approach taken by cri-o and kubelet, inspired by systemd's `conf.d` convention. +1. **Schema versioning** — the `version` field is reserved but not acted on. Should the parser reject files with `version > 1`, or just warn? Define this before the first stable release. +2. **Directory-based config (`conf.d` pattern)** — a `--config-dir` flag that globs all `*.toml` files in a directory, sorts them alphabetically, and deep-merges them in order (later files win per key). CLI/env overrides still sit above everything. This maps cleanly to Kubernetes: a base `ConfigMap` as `10-base.toml`, driver config as `20-kubernetes.toml`, and credentials from a projected `Secret` as `90-credentials.toml` — all mounted into the same directory without a monolithic file. This is the approach taken by cri-o and kubelet, inspired by systemd's `conf.d` convention. - Deferred to a follow-on: the single `--config` file is sufficient for the current schema, and the directory loader can be added without changing the file schema. Before implementing, three design decisions must be settled: (a) whether `--config` and `--config-dir` are mutually exclusive or composable (and if so which takes lower precedence); (b) whether a later file's array value (for example `credential_drivers`) replaces or appends — replace is simpler and less surprising; (c) `deny_unknown_fields` validation must apply to the final merged result rather than each individual file, since partial drop-in files won't contain all sections. -2. **OIDC secret hygiene (revisit)** — `database_url` is excluded from the file schema (resolved). Schema version 2 allows the listed OIDC fields because they are identifiers, not credentials. If we add OIDC fields that *are* credentials in the future (e.g. a client secret for confidential-client flows), they should join the env-only list at that point. Re-evaluate once the OIDC surface stabilises. + Deferred to a follow-on: the single `--config` file is sufficient for v1, and the directory loader can be added without any schema changes. Before implementing, three design decisions must be settled: (a) whether `--config` and `--config-dir` are mutually exclusive or composable (and if so which takes lower precedence); (b) whether a later file's array value (e.g. `compute_drivers`) replaces or appends — replace is simpler and less surprising; (c) `deny_unknown_fields` validation must apply to the final merged result rather than each individual file, since partial drop-in files won't contain all sections. +3. **OIDC secret hygiene (revisit)** — `database_url` is excluded from the file schema (resolved). OIDC settings are allowed for v1 since the listed fields are identifiers, not credentials. If we add OIDC fields that *are* credentials in the future (e.g. a client secret for confidential-client flows), they should join the env-only list at that point. Re-evaluate once the OIDC surface stabilises. From 326f95e8cf14d0c08da08ac9e14f4b34465746f4 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 10 Sep 2026 18:46:07 -0700 Subject: [PATCH 02/19] feat(docker): rotate launch-scoped authentication Signed-off-by: Drew Newberry --- .../openshell-driver-docker/src/isolation.rs | 51 ++-- crates/openshell-driver-docker/src/lib.rs | 227 +++++++++++++----- crates/openshell-driver-docker/src/tests.rs | 45 ++-- 3 files changed, 227 insertions(+), 96 deletions(-) diff --git a/crates/openshell-driver-docker/src/isolation.rs b/crates/openshell-driver-docker/src/isolation.rs index 08211cf9c8..5308f2357d 100644 --- a/crates/openshell-driver-docker/src/isolation.rs +++ b/crates/openshell-driver-docker/src/isolation.rs @@ -12,23 +12,24 @@ use std::net::IpAddr; use std::path::PathBuf; use openshell_isolation_interface::boundary_protocol::{ - BoundaryClientTls, BoundaryConfig, BoundaryListener, BoundaryServerTls, BoundaryTopology, - BoundaryTransport, + BoundaryConfig, BoundaryListener, BoundaryTopology, GatewayVerificationKey, + SandboxTlsClientConfig, SandboxTlsServerConfig, SandboxTransport, }; use openshell_isolation_interface::contract::{DriverFenceEvidence, ResolvedWorkloadIdentity}; /// Driver-owned inputs that bind one Docker container to one boundary. pub struct DockerBoundarySpec { pub boundary_id: String, - pub bootstrap_token: String, pub generation: String, - pub session_epoch: String, + pub session_id: openshell_core::SandboxSessionId, + pub gateway_id: String, + pub verification_keys: Vec, pub container_id: String, pub image_identity: String, pub listener_socket: PathBuf, pub control_socket: PathBuf, - pub sandbox_tls: BoundaryServerTls, - pub supervisor_tls: BoundaryClientTls, + pub sandbox_tls: SandboxTlsServerConfig, + pub supervisor_tls: SandboxTlsClientConfig, pub host_gateway_ip: Option, pub workload_identity: ResolvedWorkloadIdentity, pub child_env: HashMap, @@ -58,13 +59,13 @@ impl DockerBoundarySpec { boundary_config: BoundaryConfig { boundary_id: self.boundary_id.clone(), generation: self.generation.clone(), - session_epoch: self.session_epoch.clone(), - bootstrap_token: self.bootstrap_token.clone(), + session_id: self.session_id, + gateway_id: self.gateway_id, + verification_keys: self.verification_keys, listener: BoundaryListener::Unix { socket_path: self.listener_socket, tls: self.sandbox_tls, }, - multiplexed: true, resource_claims: resource_claims.clone(), resource_claim_files: BTreeMap::new(), workload_identity: self.workload_identity.clone(), @@ -74,17 +75,15 @@ impl DockerBoundarySpec { topology: BoundaryTopology { boundary_id: self.boundary_id, generation: self.generation, - session_epoch: self.session_epoch, + session_id: self.session_id, workload_identity: self.workload_identity, - transport: BoundaryTransport::Unix { + transport: SandboxTransport::Unix { socket_path: self.control_socket, - tls: self.supervisor_tls, }, - multiplexed: true, + tls: self.supervisor_tls, host_gateway_ip: self.host_gateway_ip, resource_claims, driver_fence, - bootstrap_token: self.bootstrap_token, }, } } @@ -96,27 +95,31 @@ mod tests { #[test] fn provisioning_binds_container_and_image_claims() { - let tls = openshell_isolation_interface::boundary_protocol::generate_boundary_mutual_tls_material() - .unwrap(); + let session_id = openshell_core::SandboxSessionId::new(); + let tls = openshell_isolation_interface::boundary_protocol::generate_sandbox_tls_material( + session_id, + ) + .unwrap(); let provisioned = DockerBoundarySpec { boundary_id: "sandbox-1".to_string(), - bootstrap_token: "a".repeat(64), generation: "generation-1".to_string(), - session_epoch: "epoch-1".to_string(), + session_id, + gateway_id: "gateway-1".to_string(), + verification_keys: vec![GatewayVerificationKey { + key_id: "key-1".to_string(), + public_key_pem: "public-key".to_string(), + }], container_id: "sha256:container".to_string(), image_identity: "sha256:image".to_string(), listener_socket: PathBuf::from("/run/openshell/boundary/control.sock"), control_socket: PathBuf::from("/host/control.sock"), - sandbox_tls: BoundaryServerTls { + sandbox_tls: SandboxTlsServerConfig { certificate_chain_path: PathBuf::from("/run/openshell/boundary/server.crt"), private_key_path: PathBuf::from("/run/openshell/boundary/server.key"), - client_ca_certificate_path: PathBuf::from("/run/openshell/boundary/client-ca.crt"), }, - supervisor_tls: BoundaryClientTls { + supervisor_tls: SandboxTlsClientConfig { server_name: tls.server_name, - ca_certificate_pem: tls.ca_certificate_pem, - certificate_chain_pem: tls.supervisor_certificate_pem, - private_key_pem: tls.supervisor_private_key_pem, + trust_anchor_pem: tls.trust_anchor_pem, }, host_gateway_ip: Some(IpAddr::from([127, 0, 0, 1])), workload_identity: ResolvedWorkloadIdentity::new( diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 397f21f9a3..3e0d70b0d4 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -61,7 +61,8 @@ use openshell_core::proto_struct::{ }; use openshell_core::{Error, Result as CoreResult}; use openshell_isolation_interface::boundary_protocol::{ - BoundaryClientTls, BoundaryServerTls, BoundaryTopology, generate_boundary_mutual_tls_material, + BoundaryConfig, BoundaryTopology, GatewayVerificationKey, SandboxTlsClientConfig, + SandboxTlsServerConfig, generate_sandbox_tls_material, }; use openshell_isolation_interface::contract::ResolvedWorkloadIdentity; use opentelemetry::trace::TraceContextExt as _; @@ -106,7 +107,6 @@ const BOUNDARY_CONFIG_MOUNT_PATH: &str = "/.openshell/channel/sandbox/bootstrap. const BOUNDARY_SOCKET_MOUNT_PATH: &str = "/.openshell/channel/sandbox/control.sock"; const BOUNDARY_CERTIFICATE_MOUNT_PATH: &str = "/.openshell/channel/sandbox/server.crt"; const BOUNDARY_PRIVATE_KEY_MOUNT_PATH: &str = "/.openshell/channel/sandbox/server.key"; -const BOUNDARY_CLIENT_CA_MOUNT_PATH: &str = "/.openshell/channel/sandbox/client-ca.crt"; const SUPERVISOR_STATE_MOUNT_PATH: &str = "/.openshell/channel/supervisor"; const DRIVER_ADMITTED_BACKEND: &str = "docker"; const LABEL_ISOLATION_TOPOLOGY: &str = "openshell.ai/isolation-topology"; @@ -122,7 +122,7 @@ const WORKSPACE_ROOT_FILE: &str = "workspace-root"; const BOUNDARY_CONFIG_FILE: &str = "boundary-bootstrap.json"; const BOUNDARY_CERTIFICATE_FILE: &str = "boundary-server.crt"; const BOUNDARY_PRIVATE_KEY_FILE: &str = "boundary-server.key"; -const BOUNDARY_CLIENT_CA_FILE: &str = "boundary-client-ca.crt"; +const SUPERVISOR_AUTH_BUNDLE_FILE: &str = "supervisor-auth.json"; const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; const DOCKER_NETWORK_DRIVER: &str = "bridge"; @@ -1001,17 +1001,16 @@ impl DockerComputeDriver { } fn validate_sandbox_auth(sandbox: &DriverSandbox) -> Result<(), Status> { - let token_present = sandbox + let authentication = sandbox .spec .as_ref() - .is_some_and(|spec| !spec.sandbox_token.trim().is_empty()); - if token_present { - return Ok(()); - } - - Err(Status::failed_precondition( - "docker sandboxes require gateway JWT auth; configure [openshell.gateway.gateway_jwt]", - )) + .filter(|spec| !spec.launch_authentication.is_empty()) + .ok_or_else(|| { + Status::failed_precondition( + "docker sandboxes require launch-scoped gateway authentication", + ) + })?; + decode_docker_launch_authentication(&authentication.launch_authentication).map(|_| ()) } fn validate_gpu_request( @@ -2124,14 +2123,19 @@ impl DockerComputeDriver { &self, sandbox_id: &str, sandbox_name: &str, + launch_authentication: &[u8], ) -> Result { let span_status = openshell_otel::ErrorStatusGuard::current(); require_sandbox_identifier(sandbox_id, sandbox_name)?; self.lifecycle_event_fences .clear_stop(sandbox_id, sandbox_name); self.lifecycle_event_fences.begin_start(sandbox_id); - let result = - Box::pin(self.start_sandbox_with_lifecycle_fence(sandbox_id, sandbox_name)).await; + let result = Box::pin(self.start_sandbox_with_lifecycle_fence( + sandbox_id, + sandbox_name, + launch_authentication, + )) + .await; self.lifecycle_event_fences.finish_start(sandbox_id); span_status.finish(result) } @@ -2140,6 +2144,7 @@ impl DockerComputeDriver { &self, sandbox_id: &str, sandbox_name: &str, + launch_authentication: &[u8], ) -> Result { let Some(container) = self .find_managed_container_summary(sandbox_id, sandbox_name) @@ -2184,6 +2189,12 @@ impl DockerComputeDriver { .as_ref() .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) .map_or(sandbox_id, String::as_str); + refresh_docker_boundary_authentication( + resolved_sandbox_id, + &self.config, + launch_authentication, + ) + .await?; let Some(topology) = read_docker_boundary_topology(resolved_sandbox_id, &self.config).await? else { @@ -2219,13 +2230,6 @@ impl DockerComputeDriver { "read Docker sandbox channel private key for restart: {error}" )) })?; - let boundary_client_ca = tokio::fs::read(boundary_directory.join(BOUNDARY_CLIENT_CA_FILE)) - .await - .map_err(|error| { - Status::failed_precondition(format!( - "read Docker sandbox channel client CA for restart: {error}" - )) - })?; let workspace_root = tokio::fs::read_to_string( docker_boundary_state_dir_by_id(resolved_sandbox_id, &self.config)? .join(WORKSPACE_ROOT_FILE), @@ -2245,7 +2249,6 @@ impl DockerComputeDriver { DockerSandboxTls { certificate: &boundary_certificate, private_key: &boundary_private_key, - client_ca: &boundary_client_ca, }, &workspace_root, ) @@ -3083,6 +3086,7 @@ impl ComputeDriver for DockerComputeDriver { self, &request.sandbox_id, &request.sandbox_name, + &request.launch_authentication, )) .await? { @@ -3940,7 +3944,6 @@ fn append_docker_archive_file( struct DockerSandboxTls<'a> { certificate: &'a [u8], private_key: &'a [u8], - client_ca: &'a [u8], } fn docker_sandbox_bundle_archive( @@ -3990,10 +3993,6 @@ fn docker_sandbox_bundle_archive( ".openshell/channel/sandbox/server.key", boundary_tls.private_key, ), - ( - ".openshell/channel/sandbox/client-ca.crt", - boundary_tls.client_ca, - ), ] { append_docker_archive_file( &mut archive, @@ -4051,6 +4050,43 @@ async fn stage_docker_sandbox_bundle( .map_err(|error| Status::internal(format!("stage Docker sandbox bundle: {error}"))) } +fn decode_docker_launch_authentication( + encoded: &[u8], +) -> Result { + let authentication = + serde_json::from_slice::(encoded) + .map_err(|error| { + Status::failed_precondition(format!( + "decode Docker sandbox launch authentication: {error}" + )) + })?; + authentication.validate().map_err(|error| { + Status::failed_precondition(format!( + "validate Docker sandbox launch authentication: {error}" + )) + })?; + Ok(authentication) +} + +fn gateway_verification_keys( + keys: &[openshell_core::jwt::SessionVerificationKey], +) -> Result, Status> { + keys.iter() + .map(|key| { + String::from_utf8(key.public_key_pem.clone()) + .map(|public_key_pem| GatewayVerificationKey { + key_id: key.key_id.clone(), + public_key_pem, + }) + .map_err(|error| { + Status::failed_precondition(format!( + "Docker sandbox verification key is not UTF-8 PEM: {error}" + )) + }) + }) + .collect() +} + async fn prepare_docker_boundary_files( docker: &Docker, sandbox: &DriverSandbox, @@ -4062,32 +4098,39 @@ async fn prepare_docker_boundary_files( let directory = docker_boundary_state_dir(sandbox, config)?; let workspace_root = driver_mounts::resolve_oci_workspace_root(&image.working_dir) .map_err(Status::failed_precondition)?; - let bootstrap_token = random_boundary_token(); + let launch_authentication = sandbox + .spec + .as_ref() + .filter(|spec| !spec.launch_authentication.is_empty()) + .ok_or_else(|| { + Status::failed_precondition("Docker sandbox launch authentication is required") + }) + .and_then(|spec| decode_docker_launch_authentication(&spec.launch_authentication))?; let host_gateway_ip = Some(match config.gateway_route { DockerGatewayRoute::Bridge { bind_address, .. } => bind_address.ip(), DockerGatewayRoute::HostGateway => IpAddr::V4(Ipv4Addr::LOCALHOST), }); - let tls = generate_boundary_mutual_tls_material() + let session_id = launch_authentication.supervisor.session_id; + let tls = generate_sandbox_tls_material(session_id) .map_err(|error| Status::internal(format!("generate Docker boundary TLS: {error}")))?; + let verification_keys = gateway_verification_keys(&launch_authentication.verification_keys)?; let provisioning = isolation::DockerBoundarySpec { boundary_id: sandbox.id.clone(), - bootstrap_token, generation: random_boundary_token(), - session_epoch: random_boundary_token(), + session_id, + gateway_id: launch_authentication.gateway_id, + verification_keys, container_id: container_id.to_string(), image_identity: image.id.clone(), listener_socket: PathBuf::from(BOUNDARY_SOCKET_MOUNT_PATH), control_socket: PathBuf::from(BOUNDARY_SOCKET_MOUNT_PATH), - sandbox_tls: BoundaryServerTls { + sandbox_tls: SandboxTlsServerConfig { certificate_chain_path: PathBuf::from(BOUNDARY_CERTIFICATE_MOUNT_PATH), private_key_path: PathBuf::from(BOUNDARY_PRIVATE_KEY_MOUNT_PATH), - client_ca_certificate_path: PathBuf::from(BOUNDARY_CLIENT_CA_MOUNT_PATH), }, - supervisor_tls: BoundaryClientTls { + supervisor_tls: SandboxTlsClientConfig { server_name: tls.server_name.clone(), - ca_certificate_pem: tls.ca_certificate_pem.clone(), - certificate_chain_pem: tls.supervisor_certificate_pem.clone(), - private_key_pem: tls.supervisor_private_key_pem.clone(), + trust_anchor_pem: tls.trust_anchor_pem.clone(), }, host_gateway_ip, workload_identity: workload_identity.clone(), @@ -4101,17 +4144,12 @@ async fn prepare_docker_boundary_files( write_docker_boundary_file(&directory.join(BOUNDARY_CONFIG_FILE), &boundary_config).await?; write_docker_boundary_file( &directory.join(BOUNDARY_CERTIFICATE_FILE), - tls.sandbox_certificate_pem.as_bytes(), + tls.certificate_chain_pem.as_bytes(), ) .await?; write_docker_boundary_file( &directory.join(BOUNDARY_PRIVATE_KEY_FILE), - tls.sandbox_private_key_pem.as_bytes(), - ) - .await?; - write_docker_boundary_file( - &directory.join(BOUNDARY_CLIENT_CA_FILE), - tls.ca_certificate_pem.as_bytes(), + tls.private_key_pem.as_bytes(), ) .await?; stage_docker_sandbox_bundle( @@ -4121,9 +4159,8 @@ async fn prepare_docker_boundary_files( workload_identity, &boundary_config, DockerSandboxTls { - certificate: tls.sandbox_certificate_pem.as_bytes(), - private_key: tls.sandbox_private_key_pem.as_bytes(), - client_ca: tls.ca_certificate_pem.as_bytes(), + certificate: tls.certificate_chain_pem.as_bytes(), + private_key: tls.private_key_pem.as_bytes(), }, &workspace_root, ) @@ -4133,6 +4170,13 @@ async fn prepare_docker_boundary_files( .descriptor(DRIVER_ADMITTED_BACKEND) .map_err(|error| Status::internal(error.to_string()))?; write_docker_boundary_file(&directory.join(TOPOLOGY_PAYLOAD_FILE), &descriptor.payload).await?; + let supervisor_auth = serde_json::to_vec(&launch_authentication.supervisor) + .map_err(|error| Status::internal(format!("encode Docker supervisor auth: {error}")))?; + write_docker_boundary_file( + &directory.join(SUPERVISOR_AUTH_BUNDLE_FILE), + &supervisor_auth, + ) + .await?; let main_process_spec = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec( sandbox.spec.as_ref(), ) @@ -4158,14 +4202,14 @@ async fn docker_supervisor_bundle_archive( let topology = tokio::fs::read(directory.join(TOPOLOGY_PAYLOAD_FILE)) .await .map_err(|error| Status::internal(format!("read Docker topology payload: {error}")))?; - let token = tokio::fs::read(sandbox_token_host_path(sandbox, config)?) + let auth_bundle = tokio::fs::read(directory.join(SUPERVISOR_AUTH_BUNDLE_FILE)) .await .map_err(|error| { - Status::failed_precondition(format!("read Docker sandbox JWT: {error}")) + Status::failed_precondition(format!("read Docker supervisor auth bundle: {error}")) })?; - if token.iter().all(u8::is_ascii_whitespace) { + if auth_bundle.is_empty() { return Err(Status::failed_precondition( - "Docker supervisor requires a sandbox JWT", + "Docker supervisor requires launch authentication", )); } let mut archive = tar::Builder::new(Vec::new()); @@ -4186,11 +4230,11 @@ async fn docker_supervisor_bundle_archive( )?; append_docker_archive_file( &mut archive, - ".openshell/channel/supervisor/sandbox.jwt", + ".openshell/channel/supervisor/auth.json", 0o600, SUPERVISOR_UID, SUPERVISOR_GID, - &token, + &auth_bundle, )?; if let Some(tls) = &config.guest_tls { append_docker_archive_directory( @@ -4226,6 +4270,75 @@ async fn docker_supervisor_bundle_archive( .map_err(|error| Status::internal(format!("finish Docker supervisor archive: {error}"))) } +async fn refresh_docker_boundary_authentication( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, + encoded_authentication: &[u8], +) -> Result<(), Status> { + let authentication = decode_docker_launch_authentication(encoded_authentication)?; + let directory = docker_boundary_state_dir_by_id(sandbox_id, config)?; + let mut boundary_config = serde_json::from_slice::( + &tokio::fs::read(directory.join(BOUNDARY_CONFIG_FILE)) + .await + .map_err(|error| { + Status::failed_precondition(format!( + "read Docker sandbox bootstrap for authentication rotation: {error}" + )) + })?, + ) + .map_err(|error| { + Status::failed_precondition(format!( + "decode Docker sandbox bootstrap for authentication rotation: {error}" + )) + })?; + let Some(mut topology) = read_docker_boundary_topology(sandbox_id, config).await? else { + return Err(Status::failed_precondition( + "Docker sandbox topology is missing during authentication rotation", + )); + }; + let session_id = authentication.supervisor.session_id; + let tls = generate_sandbox_tls_material(session_id) + .map_err(|error| Status::internal(format!("rotate Docker boundary TLS: {error}")))?; + boundary_config.session_id = session_id; + boundary_config.gateway_id = authentication.gateway_id; + boundary_config.verification_keys = + gateway_verification_keys(&authentication.verification_keys)?; + topology.session_id = session_id; + topology.tls = SandboxTlsClientConfig { + server_name: tls.server_name, + trust_anchor_pem: tls.trust_anchor_pem, + }; + let encoded_boundary_config = boundary_config + .encode() + .map_err(|error| Status::internal(error.to_string()))?; + let descriptor = topology + .descriptor(DRIVER_ADMITTED_BACKEND) + .map_err(|error| Status::internal(error.to_string()))?; + let supervisor_auth = serde_json::to_vec(&authentication.supervisor) + .map_err(|error| Status::internal(format!("encode Docker supervisor auth: {error}")))?; + write_docker_boundary_file( + &directory.join(BOUNDARY_CONFIG_FILE), + &encoded_boundary_config, + ) + .await?; + write_docker_boundary_file( + &directory.join(BOUNDARY_CERTIFICATE_FILE), + tls.certificate_chain_pem.as_bytes(), + ) + .await?; + write_docker_boundary_file( + &directory.join(BOUNDARY_PRIVATE_KEY_FILE), + tls.private_key_pem.as_bytes(), + ) + .await?; + write_docker_boundary_file(&directory.join(TOPOLOGY_PAYLOAD_FILE), &descriptor.payload).await?; + write_docker_boundary_file( + &directory.join(SUPERVISOR_AUTH_BUNDLE_FILE), + &supervisor_auth, + ) + .await +} + async fn read_docker_boundary_topology( sandbox_id: &str, config: &DockerDriverRuntimeConfig, @@ -4375,7 +4488,7 @@ async fn spawn_docker_control_process( ) .await; let topology_path = format!("{SUPERVISOR_STATE_MOUNT_PATH}/topology.payload"); - let token_path = format!("{SUPERVISOR_STATE_MOUNT_PATH}/sandbox.jwt"); + let auth_bundle_path = format!("{SUPERVISOR_STATE_MOUNT_PATH}/auth.json"); let mut environment = vec![ format!( "{}={DRIVER_ADMITTED_BACKEND}", @@ -4392,10 +4505,6 @@ async fn spawn_docker_control_process( ), format!("{}={}", openshell_core::sandbox_env::SANDBOX_ID, sandbox.id), format!("{}={}", openshell_core::sandbox_env::SANDBOX, sandbox.name), - format!( - "{}={token_path}", - openshell_core::sandbox_env::SANDBOX_TOKEN_FILE - ), format!( "{}=/run/openshell/ssh.sock", openshell_core::sandbox_env::SSH_SOCKET_PATH @@ -4465,6 +4574,8 @@ async fn spawn_docker_control_process( format!("--topology-backend-name={}", descriptor.backend_name), "--topology-payload-file".to_string(), topology_path.clone(), + "--auth-bundle-file".to_string(), + auth_bundle_path, "--workdir".to_string(), workspace_root, format!("--health-socket-path={SUPERVISOR_HEALTH_SOCKET_PATH}"), diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index ba86bf888a..3873245418 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -7,6 +7,10 @@ use openshell_core::driver_utils::{ LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, }; +use openshell_core::jwt::{ + CredentialEpoch, SandboxLaunchAuthentication, SecretJwt, SessionVerificationKey, + SupervisorAuthBundle, +}; use openshell_core::progress::{ PROGRESS_ACTIVE_DETAIL_KEY, PROGRESS_ACTIVE_STEP_KEY, PROGRESS_COMPLETE_LABEL_KEY, PROGRESS_COMPLETE_STEP_KEY, PROGRESS_STEP_PULLING_IMAGE, PROGRESS_STEP_REQUESTING_SANDBOX, @@ -22,6 +26,25 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::sync::Arc; use tempfile::TempDir; +fn test_launch_authentication() -> Vec { + serde_json::to_vec(&SandboxLaunchAuthentication { + supervisor: SupervisorAuthBundle { + session_id: openshell_core::SandboxSessionId::new(), + gateway_token: SecretJwt::parse("gateway.token.value").unwrap(), + gateway_expires_at: i64::MAX, + sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(), + sandbox_expires_at: i64::MAX, + credential_epoch: CredentialEpoch::new(1).unwrap(), + }, + gateway_id: "gateway-test".to_string(), + verification_keys: vec![SessionVerificationKey { + key_id: "test-key".to_string(), + public_key_pem: b"public-key".to_vec(), + }], + }) + .unwrap() +} + fn test_sandbox() -> DriverSandbox { // Mirrors the gateway-supplied request: the public `Sandbox` API no // longer carries `namespace`, so the gateway elides the field and the @@ -47,7 +70,7 @@ fn test_sandbox() -> DriverSandbox { tty: false, await_main_process_attachment: false, workload_identity: None, - launch_authentication: Vec::new(), + launch_authentication: test_launch_authentication(), }), status: None, workspace: String::new(), @@ -606,7 +629,7 @@ async fn tracing_direct_start_exports_a_docker_start_span() { let subscriber = tracing_subscriber::registry().with(otel_tracing::TRACING.layer(&provider)); let driver = test_driver_with_config(runtime_config()); - Box::pin(DockerComputeDriver::start_sandbox(&driver, "", "").with_subscriber(subscriber)) + Box::pin(DockerComputeDriver::start_sandbox(&driver, "", "", &[]).with_subscriber(subscriber)) .await .expect_err("missing identifier should fail"); provider.force_flush().unwrap(); @@ -1443,7 +1466,6 @@ fn sandbox_bundle_prepares_only_the_driver_managed_workspace() { DockerSandboxTls { certificate: b"server-cert", private_key: b"server-key", - client_ca: b"client-ca", }, &identity, driver_mounts::DEFAULT_WORKSPACE_ROOT, @@ -1473,7 +1495,6 @@ fn sandbox_bundle_prepares_only_the_driver_managed_workspace() { DockerSandboxTls { certificate: b"server-cert", private_key: b"server-key", - client_ca: b"client-ca", }, &identity, "/workspace/project", @@ -1490,7 +1511,7 @@ fn sandbox_bundle_prepares_only_the_driver_managed_workspace() { } #[test] -fn sandbox_bundle_stages_private_mutual_tls_material() { +fn sandbox_bundle_stages_private_tls_server_material() { let identity = test_workload_identity(); let archive = docker_sandbox_bundle_archive( b"sandbox-binary", @@ -1498,7 +1519,6 @@ fn sandbox_bundle_stages_private_mutual_tls_material() { DockerSandboxTls { certificate: b"server-cert", private_key: b"server-key", - client_ca: b"client-ca", }, &identity, driver_mounts::DEFAULT_WORKSPACE_ROOT, @@ -1524,7 +1544,6 @@ fn sandbox_bundle_stages_private_mutual_tls_material() { for path in [ ".openshell/channel/sandbox/server.crt", ".openshell/channel/sandbox/server.key", - ".openshell/channel/sandbox/client-ca.crt", ] { assert_eq!( entries.get(Path::new(path)), @@ -2511,24 +2530,22 @@ fn validate_sandbox_rejects_template_errors_before_device_config() { } #[test] -fn validate_sandbox_auth_requires_gateway_token() { +fn validate_sandbox_auth_requires_launch_authentication() { let mut sandbox = test_sandbox(); - sandbox.spec.as_mut().unwrap().sandbox_token.clear(); + sandbox.spec.as_mut().unwrap().launch_authentication.clear(); let err = DockerComputeDriver::validate_sandbox_auth(&sandbox).unwrap_err(); assert_eq!(err.code(), tonic::Code::FailedPrecondition); assert_eq!( err.message(), - "docker sandboxes require gateway JWT auth; configure [openshell.gateway.gateway_jwt]" + "docker sandboxes require launch-scoped gateway authentication" ); } #[test] -fn validate_sandbox_auth_accepts_gateway_token() { - let mut sandbox = test_sandbox(); - sandbox.spec.as_mut().unwrap().sandbox_token = "secret.jwt.value".to_string(); - +fn validate_sandbox_auth_accepts_launch_authentication() { + let sandbox = test_sandbox(); DockerComputeDriver::validate_sandbox_auth(&sandbox).unwrap(); } From 7398c852dce490412c54fbbb283501b72a30f4ae Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 11 Sep 2026 10:00:47 -0700 Subject: [PATCH 03/19] refactor(docker): use sandbox backend protocol Signed-off-by: Drew Newberry --- Cargo.lock | 1 + crates/openshell-driver-docker/Cargo.toml | 1 + crates/openshell-driver-docker/src/isolation.rs | 11 +++++------ crates/openshell-driver-docker/src/lib.rs | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 07a646739e..ce581c6bc3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4104,6 +4104,7 @@ dependencies = [ "openshell-isolation-interface", "openshell-otel", "openshell-otel-test-support", + "openshell-sandbox-backend", "opentelemetry", "opentelemetry_sdk", "prost-types", diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index 38051327aa..e9751f7763 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -17,6 +17,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } openshell-isolation-interface = { path = "../openshell-isolation-interface" } +openshell-sandbox-backend = { path = "../openshell-sandbox-backend" } openshell-otel = { path = "../openshell-otel" } opentelemetry = { workspace = true } diff --git a/crates/openshell-driver-docker/src/isolation.rs b/crates/openshell-driver-docker/src/isolation.rs index 5308f2357d..4dff02789b 100644 --- a/crates/openshell-driver-docker/src/isolation.rs +++ b/crates/openshell-driver-docker/src/isolation.rs @@ -11,11 +11,11 @@ use std::collections::{BTreeMap, HashMap}; use std::net::IpAddr; use std::path::PathBuf; -use openshell_isolation_interface::boundary_protocol::{ +use openshell_isolation_interface::contract::{DriverFenceEvidence, ResolvedWorkloadIdentity}; +use openshell_sandbox_backend::boundary_protocol::{ BoundaryConfig, BoundaryListener, BoundaryTopology, GatewayVerificationKey, SandboxTlsClientConfig, SandboxTlsServerConfig, SandboxTransport, }; -use openshell_isolation_interface::contract::{DriverFenceEvidence, ResolvedWorkloadIdentity}; /// Driver-owned inputs that bind one Docker container to one boundary. pub struct DockerBoundarySpec { @@ -96,10 +96,9 @@ mod tests { #[test] fn provisioning_binds_container_and_image_claims() { let session_id = openshell_core::SandboxSessionId::new(); - let tls = openshell_isolation_interface::boundary_protocol::generate_sandbox_tls_material( - session_id, - ) - .unwrap(); + let tls = + openshell_sandbox_backend::boundary_protocol::generate_sandbox_tls_material(session_id) + .unwrap(); let provisioned = DockerBoundarySpec { boundary_id: "sandbox-1".to_string(), generation: "generation-1".to_string(), diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 3e0d70b0d4..382df0665f 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -60,11 +60,11 @@ use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, }; use openshell_core::{Error, Result as CoreResult}; -use openshell_isolation_interface::boundary_protocol::{ +use openshell_isolation_interface::contract::ResolvedWorkloadIdentity; +use openshell_sandbox_backend::boundary_protocol::{ BoundaryConfig, BoundaryTopology, GatewayVerificationKey, SandboxTlsClientConfig, SandboxTlsServerConfig, generate_sandbox_tls_material, }; -use openshell_isolation_interface::contract::ResolvedWorkloadIdentity; use opentelemetry::trace::TraceContextExt as _; use sha2::{Digest as _, Sha256}; use std::collections::{HashMap, HashSet}; From eaa76f31817f85b00899d51d01fdd0edf73dd6c1 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 11 Sep 2026 10:42:14 -0700 Subject: [PATCH 04/19] refactor(docker): use host networking for supervisor Signed-off-by: Drew Newberry --- crates/openshell-driver-docker/README.md | 5 +++-- crates/openshell-driver-docker/src/lib.rs | 8 +++++--- crates/openshell-driver-docker/src/tests.rs | 1 - 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index b8f9cd2011..4c3cb2f964 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -34,7 +34,7 @@ mediates every supported TCP and DNS operation, attributes it to the calling binary, and sends the request across the private channel. The supervisor authorizes the request before it opens an upstream connection. Docker's absent workload network is the mandatory outer fence if mediation fails or is -bypassed. Only the supervisor companion joins the managed bridge network. +bypassed. Only the trusted supervisor companion uses the daemon host network. The driver copies trusted runtime bytes from the configured supervisor image through the Docker archive API. No workload launch depends on a host bind @@ -69,7 +69,8 @@ LSM decisions remain authoritative. | Exact non-root `user` and `group_add` | Gives sandbox and workload the same immutable UID/GID/group identity required for capability-free observation. | | `cap_drop = ALL`, no `cap_add`, no-new-privileges | Prevents either container from acquiring Linux capabilities. | | Docker default seccomp and AppArmor profiles | Retains runtime hardening; startup confirmation fails closed if nested seccomp notification is unavailable. | -| `network_mode = none` on the workload | Removes direct external routes. The supervisor companion alone has bridge networking. | +| `network_mode = none` on the workload | Removes direct external routes. | +| `network_mode = host` on the supervisor | Lets the trusted supervisor originate approved gateway and upstream connections through the daemon host network. | | `restart_policy = no` | Keeps canonical main-process exit terminal. | | `PidsLimit` | Applies the configured sandbox PID budget. Set `sandbox_pids_limit = 0` to use the runtime default. | | Private named volume | Carries a per-generation mutual-TLS sandbox/supervisor channel without sharing daemon-host paths. The sandbox consumes its server key at startup; only the supervisor receives the client key. | diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 382df0665f..15bbcb6d8c 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -114,6 +114,7 @@ const LABEL_ISOLATION_TOPOLOGY_CAPABILITY_FREE: &str = "capability-free"; const LABEL_ISOLATION_ROLE: &str = "openshell.ai/isolation-role"; const LABEL_ISOLATION_ROLE_SANDBOX: &str = "sandbox"; const LABEL_ISOLATION_ROLE_SUPERVISOR: &str = "supervisor"; +const SUPERVISOR_NETWORK_MODE: &str = "host"; const LABEL_ISOLATION_ROLE_STAGING: &str = "staging"; const LABEL_ISOLATION_ROLE_IDENTITY: &str = "identity"; const TOPOLOGY_PAYLOAD_FILE: &str = "topology.payload"; @@ -238,7 +239,6 @@ struct DockerDriverRuntimeConfig { log_level: String, sandbox_binary: Arc>, supervisor_image_id: String, - network_name: String, supervisor_grpc_endpoint: String, gateway_tls_server_name: Option, guest_tls: Option, @@ -878,7 +878,6 @@ impl DockerComputeDriver { log_level: gateway_log_level.to_string(), sandbox_binary, supervisor_image_id, - network_name, supervisor_grpc_endpoint, gateway_tls_server_name, guest_tls, @@ -4597,7 +4596,10 @@ async fn spawn_docker_control_process( start_interval: Some(SUPERVISOR_HEALTH_INTERVAL_NS), }), host_config: Some(HostConfig { - network_mode: Some(config.network_name.clone()), + // The supervisor is trusted infrastructure and originates all + // approved upstream connections. Keep it on the daemon host's + // network while the workload remains fenced by network=none. + network_mode: Some(SUPERVISOR_NETWORK_MODE.to_string()), mounts: Some(vec![Mount { target: Some(BOUNDARY_MOUNT_PATH.to_string()), source: Some(docker_channel_volume_name(sandbox, config)), diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 3873245418..16f71d30f7 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -133,7 +133,6 @@ fn runtime_config() -> DockerDriverRuntimeConfig { log_level: "info".to_string(), sandbox_binary: Arc::new(b"\x7fELFtest".to_vec()), supervisor_image_id: "sha256:supervisor-test".to_string(), - network_name: "openshell-test".to_string(), supervisor_grpc_endpoint: "https://host.openshell.internal:8443".to_string(), gateway_tls_server_name: None, guest_tls: Some(DockerGuestTlsPaths { From 2ace69d61ea33038dd899209d8c34627a2c0e24b Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 11 Sep 2026 12:12:46 -0700 Subject: [PATCH 05/19] fix(docker): preserve host gateway alias resolution Signed-off-by: Drew Newberry --- crates/openshell-driver-docker/README.md | 5 +++++ crates/openshell-driver-docker/src/lib.rs | 15 +++++++++++---- crates/openshell-driver-docker/src/tests.rs | 14 ++++++++++++++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 4c3cb2f964..79ac2e76e0 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -136,6 +136,11 @@ gateway. Docker Desktop and compatible VM-backed daemons use Docker's `host-gateway` route. A configured HTTPS server certificate must include the endpoint host in its subject alternative names. +The driver pins a concrete managed-bridge address in the sandbox descriptor. +For Docker's special `host-gateway` route, it leaves the address unpinned so +the supervisor resolves the driver-injected alias in its own container. This +avoids treating the Docker VM's loopback as the desktop host. + The supervisor owns these security-critical variables: - `OPENSHELL_ENDPOINT` diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 15bbcb6d8c..4ffce52d71 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -4105,10 +4105,7 @@ async fn prepare_docker_boundary_files( Status::failed_precondition("Docker sandbox launch authentication is required") }) .and_then(|spec| decode_docker_launch_authentication(&spec.launch_authentication))?; - let host_gateway_ip = Some(match config.gateway_route { - DockerGatewayRoute::Bridge { bind_address, .. } => bind_address.ip(), - DockerGatewayRoute::HostGateway => IpAddr::V4(Ipv4Addr::LOCALHOST), - }); + let host_gateway_ip = docker_boundary_host_gateway_ip(&config.gateway_route); let session_id = launch_authentication.supervisor.session_id; let tls = generate_sandbox_tls_material(session_id) .map_err(|error| Status::internal(format!("generate Docker boundary TLS: {error}")))?; @@ -5303,6 +5300,16 @@ fn docker_supervisor_host_alias(route: &DockerGatewayRoute) -> String { } } +fn docker_boundary_host_gateway_ip(route: &DockerGatewayRoute) -> Option { + match route { + DockerGatewayRoute::Bridge { bind_address } => Some(bind_address.ip()), + // Docker resolves this special alias inside the supervisor container. + // Pinning it to loopback would target the daemon VM rather than the + // desktop host on Docker Desktop and compatible runtimes. + DockerGatewayRoute::HostGateway => None, + } +} + fn docker_network_name(config: &DockerComputeConfig) -> String { let name = config.network_name.trim(); if name.is_empty() { diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 16f71d30f7..f34015c4aa 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -1214,6 +1214,20 @@ fn docker_supervisor_alias_matches_the_trusted_gateway_route() { ); } +#[test] +fn docker_boundary_pins_only_concrete_host_gateway_addresses() { + assert_eq!( + docker_boundary_host_gateway_ip(&DockerGatewayRoute::Bridge { + bind_address: "172.20.0.4:17670".parse().unwrap(), + }), + Some(IpAddr::V4(Ipv4Addr::new(172, 20, 0, 4))) + ); + assert_eq!( + docker_boundary_host_gateway_ip(&DockerGatewayRoute::HostGateway), + None + ); +} + #[test] fn parse_optional_host_gateway_ip_rejects_invalid_values() { assert_eq!(parse_optional_host_gateway_ip("").unwrap(), None); From dccdcbbabacd929baebd3a820166e06b25fbcfdc Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 11 Sep 2026 13:08:02 -0700 Subject: [PATCH 06/19] feat(docker): use separate sandbox and supervisor images Signed-off-by: Drew Newberry --- crates/openshell-driver-docker/README.md | 11 ++-- crates/openshell-driver-docker/src/lib.rs | 65 ++++++++++++--------- e2e/configs/gateway/docker.toml | 1 + e2e/run.sh | 44 ++++++++++++-- e2e/with-docker-gateway.sh | 70 +++++++++++++++++++++++ 5 files changed, 153 insertions(+), 38 deletions(-) diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 79ac2e76e0..803da311c4 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -118,11 +118,12 @@ openshell sandbox create \ ## Runtime Image -`supervisor_image` must contain `/openshell-sandbox` and -`/openshell-supervisor`. The driver extracts the sandbox binary as bytes and -stages it into the stopped workload. It starts the supervisor binary directly -in the companion container. Release and gateway image builds bake a matching -supervisor image tag into the binary. +`sandbox_runtime_image` contains the statically linked musl +`/openshell-sandbox` binary. The driver extracts that binary as bytes and +stages it into the stopped workload. `supervisor_image` contains the +dynamically linked glibc `/openshell-supervisor` binary that runs in the +host-networked supervisor container. Release and gateway image builds bake +matching image tags into the binary. ## Callback and TLS diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 4ffce52d71..b41ac6d2b1 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -26,12 +26,10 @@ use futures::{Stream, StreamExt}; use openshell_core::config::{DEFAULT_SANDBOX_PIDS_LIMIT, DEFAULT_STOP_TIMEOUT_SECS}; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ - CONDITION_EXITED, CONDITION_RUNTIME_RESTART, CONDITION_WORKSPACE_VALIDATION_FAILED, - GatewayCallbackRoute, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, - LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, - SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED, SUPERVISOR_IMAGE_BINARY_PATH, - extract_first_tar_entry, gateway_callback_endpoint, supervisor_image_should_refresh, - temp_extract_container_name, validate_linux_elf_binary, write_cache_binary_atomic, + CONDITION_EXITED, CONDITION_RUNTIME_RESTART, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, + LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, + GatewayCallbackRoute, SANDBOX_RUNTIME_IMAGE_BINARY_PATH, extract_first_tar_entry, + gateway_callback_endpoint, supervisor_image_should_refresh, temp_extract_container_name, }; use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, @@ -171,8 +169,10 @@ pub struct DockerComputeConfig { /// Gateway gRPC endpoint the sandbox connects back to. pub grpc_endpoint: String, - /// Image containing the trusted `openshell-sandbox` and - /// `openshell-supervisor` binaries. + /// Image containing the trusted `openshell-sandbox` binary. + pub sandbox_runtime_image: Option, + + /// Image containing the trusted `openshell-supervisor` binary. pub supervisor_image: Option, /// Host-side CA certificate for Docker sandbox mTLS. @@ -209,6 +209,7 @@ impl Default for DockerComputeConfig { image_pull_policy: String::new(), sandbox_namespace: "default".to_string(), grpc_endpoint: String::new(), + sandbox_runtime_image: None, supervisor_image: None, guest_tls_ca: None, guest_tls_cert: None, @@ -855,13 +856,19 @@ impl DockerComputeDriver { .clone() .unwrap_or_else(openshell_core::config::default_supervisor_image); let supervisor_image_id = - ensure_supervisor_container_image(&docker, &supervisor_image).await?; + ensure_runtime_image(&docker, &supervisor_image, "supervisor").await?; + let sandbox_runtime_image = docker_config + .sandbox_runtime_image + .clone() + .unwrap_or_else(openshell_core::config::default_sandbox_runtime_image); + let sandbox_runtime_image_id = + ensure_runtime_image(&docker, &sandbox_runtime_image, "sandbox runtime").await?; let sandbox_binary = Arc::new( - extract_supervisor_binary_bytes(&docker, &supervisor_image_id) + extract_sandbox_binary_bytes(&docker, &sandbox_runtime_image_id) .await .map_err(|error| { Error::config(format!( - "failed to load trusted sandbox binary from Docker image '{supervisor_image}': {error}" + "failed to load trusted sandbox binary from Docker image '{sandbox_runtime_image}': {error}" )) })?, ); @@ -5903,7 +5910,7 @@ fn sanitize_docker_name(value: &str) -> String { .to_string() } -async fn pull_supervisor_image(docker: &Docker, image: &str) -> CoreResult<()> { +async fn pull_runtime_image(docker: &Docker, image: &str, role: &str) -> CoreResult<()> { let mut stream = docker.create_image( Some(CreateImageOptions { from_image: Some(image.to_string()), @@ -5915,57 +5922,61 @@ async fn pull_supervisor_image(docker: &Docker, image: &str) -> CoreResult<()> { while let Some(result) = stream.next().await { result.map_err(|err| { Error::config(format!( - "failed to pull docker supervisor image '{image}': {err}", + "failed to pull Docker {role} image '{image}': {err}", )) })?; } Ok(()) } -async fn ensure_supervisor_container_image(docker: &Docker, image: &str) -> CoreResult { +async fn ensure_runtime_image(docker: &Docker, image: &str, role: &str) -> CoreResult { let local_image_present = docker.inspect_image(image).await.is_ok(); if supervisor_image_should_refresh(image) { - info!(image = image, "Refreshing mutable docker supervisor image"); - if let Err(error) = pull_supervisor_image(docker, image).await { + info!( + image = image, + role, "Refreshing mutable Docker runtime image" + ); + if let Err(error) = pull_runtime_image(docker, image, role).await { if !local_image_present { return Err(error); } warn!( image = image, error = %error, - "failed to refresh mutable Docker supervisor image; using the local image", + "failed to refresh mutable Docker runtime image; using the local image", ); } } else if !local_image_present { - pull_supervisor_image(docker, image).await?; + pull_runtime_image(docker, image, role).await?; } let inspect = docker.inspect_image(image).await.map_err(|error| { Error::config(format!( - "failed to inspect Docker supervisor image '{image}': {error}" + "failed to inspect Docker {role} image '{image}': {error}" )) })?; inspect.id.filter(|id| !id.is_empty()).ok_or_else(|| { Error::config(format!( - "Docker supervisor image '{image}' has no immutable image ID" + "Docker {role} image '{image}' has no immutable image ID" )) }) } -/// Create a short-lived container from `image`, stream out the supervisor +/// Create a short-lived container from `image`, stream out the sandbox /// binary as a tar archive, and return the untarred file bytes. The /// container is always removed, even on error paths. -async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreResult> { +async fn extract_sandbox_binary_bytes(docker: &Docker, image: &str) -> CoreResult> { let bytes = - extract_supervisor_path_archive(docker, image, SUPERVISOR_IMAGE_BINARY_PATH, true).await?; + extract_runtime_path_archive(docker, image, SANDBOX_RUNTIME_IMAGE_BINARY_PATH, true) + .await?; if !bytes.starts_with(b"\x7fELF") { return Err(Error::config(format!( - "Docker supervisor image '{image}' contains an invalid sandbox binary" + "Docker sandbox runtime image '{image}' contains an invalid sandbox binary" ))); } Ok(bytes) } -async fn extract_supervisor_path_archive( +async fn extract_runtime_path_archive( docker: &Docker, image: &str, path: &str, @@ -5981,7 +5992,7 @@ async fn extract_supervisor_path_archive( ), ContainerCreateBody { image: Some(image.to_string()), - entrypoint: Some(vec![SUPERVISOR_IMAGE_BINARY_PATH.to_string()]), + entrypoint: Some(vec![path.to_string()]), cmd: Some(Vec::new()), ..Default::default() }, @@ -6006,7 +6017,7 @@ async fn extract_supervisor_path_archive( warn!( container = container_name, error = %remove_err, - "Failed to remove supervisor extractor container", + "Failed to remove runtime image extractor container", ); } result diff --git a/e2e/configs/gateway/docker.toml b/e2e/configs/gateway/docker.toml index 63e587ef62..3e872885e1 100644 --- a/e2e/configs/gateway/docker.toml +++ b/e2e/configs/gateway/docker.toml @@ -23,5 +23,6 @@ gateway_id = "openshell-e2e" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" image_pull_policy = "if_not_present" sandbox_label = "openshell-e2e" +sandbox_runtime_image = "localhost/openshell/sandbox:e2e-vm" supervisor_image = "localhost/openshell/supervisor:e2e-vm" app_armor_profile = "Unconfined" diff --git a/e2e/run.sh b/e2e/run.sh index 8d9cee8dac..fd4eeb26ea 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -249,6 +249,16 @@ mise x -- cargo zigbuild "${cargo_jobs[@]}" \ --bin openshell-sandbox linux_sandbox_bin="${target_dir}/${linux_musl_target}/release/openshell-sandbox" +echo "==> Preparing ${linux_gateway_rust_target} build target" +mise x -- rustup target add "${linux_gateway_rust_target}" >/dev/null +echo "==> Building Linux openshell-supervisor (${linux_gateway_zig_target})" +mise x -- cargo zigbuild "${cargo_jobs[@]}" \ + --release \ + --target "${linux_gateway_zig_target}" \ + -p openshell-supervisor \ + --bin openshell-supervisor +linux_supervisor_bin="${target_dir}/${linux_gateway_rust_target}/release/openshell-supervisor" + host_gateway_bin= guest_gateway_bin= if [ "${mode}" = host ]; then @@ -259,8 +269,6 @@ if [ "${mode}" = host ]; then --features bundled-z3 host_gateway_bin="${target_dir}/debug/openshell-gateway" else - echo "==> Preparing ${linux_gateway_rust_target} build target" - mise x -- rustup target add "${linux_gateway_rust_target}" >/dev/null echo "==> Building Linux openshell-gateway (${linux_gateway_zig_target})" ( eval "$( @@ -279,7 +287,7 @@ else guest_gateway_bin="${target_dir}/${linux_gateway_rust_target}/release/openshell-gateway" fi -expected_binaries=("${host_cli_bin}" "${linux_sandbox_bin}") +expected_binaries=("${host_cli_bin}" "${linux_sandbox_bin}" "${linux_supervisor_bin}") if [ "${mode}" = host ]; then expected_binaries+=("${host_gateway_bin}") else @@ -296,14 +304,20 @@ run_parent="${ROOT}/.cache/openshell-e2e/runs" mkdir -p "${run_parent}" run_dir="$(mktemp -d "${run_parent%/}/run.XXXXXX")" if ! command -v tar >/dev/null 2>&1; then - die "tar is required to package the supervisor image" + die "tar is required to package the runtime images" fi +sandbox_runtime_image=localhost/openshell/sandbox:e2e-vm +sandbox_runtime_rootfs="${run_dir}/sandbox-runtime-rootfs" +sandbox_runtime_archive="${run_dir}/sandbox-runtime.tar" +mkdir -p "${sandbox_runtime_rootfs}" +install -m 0555 "${linux_sandbox_bin}" "${sandbox_runtime_rootfs}/openshell-sandbox" +tar -C "${sandbox_runtime_rootfs}" -cf "${sandbox_runtime_archive}" openshell-sandbox supervisor_image=localhost/openshell/supervisor:e2e-vm supervisor_rootfs="${run_dir}/supervisor-rootfs" supervisor_archive="${run_dir}/supervisor.tar" mkdir -p "${supervisor_rootfs}" -install -m 0555 "${linux_sandbox_bin}" "${supervisor_rootfs}/openshell-sandbox" -tar -C "${supervisor_rootfs}" -cf "${supervisor_archive}" openshell-sandbox +install -m 0555 "${linux_supervisor_bin}" "${supervisor_rootfs}/openshell-supervisor" +tar -C "${supervisor_rootfs}" -cf "${supervisor_archive}" openshell-supervisor child_pid= runtime_log= keep=0 @@ -402,12 +416,20 @@ if [ "${mode}" = host ]; then e2e_align_docker_host_with_cli_context docker import \ --change 'ENTRYPOINT ["/openshell-sandbox"]' \ + "${sandbox_runtime_archive}" \ + "${sandbox_runtime_image}" >/dev/null + docker import \ + --change 'ENTRYPOINT ["/openshell-supervisor"]' \ "${supervisor_archive}" \ "${supervisor_image}" >/dev/null ;; podman) podman import \ --change 'ENTRYPOINT ["/openshell-sandbox"]' \ + "${sandbox_runtime_archive}" \ + "${sandbox_runtime_image}" >/dev/null + podman import \ + --change 'ENTRYPOINT ["/openshell-supervisor"]' \ "${supervisor_archive}" \ "${supervisor_image}" >/dev/null ;; @@ -427,6 +449,7 @@ else runtime_log="${run_dir}/vm.log" guest_launcher="${run_dir}/launch-gateway.sh" guest_launcher_path=/home/openshell/.cache/openshell-e2e/bin/launch-gateway + guest_sandbox_runtime_archive_path=/home/openshell/.cache/openshell-e2e/sandbox-runtime.tar guest_supervisor_archive_path=/home/openshell/.cache/openshell-e2e/supervisor.tar config_payload="$(base64 <"${gateway_config}" | tr -d '\r\n')" jwt_signing_payload="$(base64 <"${jwt_source_dir}/signing.pem" | tr -d '\r\n')" @@ -467,12 +490,20 @@ case '${gateway_driver}' in docker) docker import \ --change 'ENTRYPOINT ["/openshell-sandbox"]' \ + "${guest_sandbox_runtime_archive_path}" \ + "${sandbox_runtime_image}" >/dev/null + docker import \ + --change 'ENTRYPOINT ["/openshell-supervisor"]' \ "${guest_supervisor_archive_path}" \ "${supervisor_image}" >/dev/null ;; podman) podman --url "unix:///run/user/\$(id -u)/podman/podman.sock" import \ --change 'ENTRYPOINT ["/openshell-sandbox"]' \ + "${guest_sandbox_runtime_archive_path}" \ + "${sandbox_runtime_image}" >/dev/null + podman --url "unix:///run/user/\$(id -u)/podman/podman.sock" import \ + --change 'ENTRYPOINT ["/openshell-supervisor"]' \ "${guest_supervisor_archive_path}" \ "${supervisor_image}" >/dev/null ;; @@ -497,6 +528,7 @@ EOF vm_args+=( --copy "${guest_gateway_bin}:/usr/local/bin/openshell-gateway" --copy "${guest_launcher}:${guest_launcher_path}" + --copy "${sandbox_runtime_archive}:${guest_sandbox_runtime_archive_path}" --copy "${supervisor_archive}:${guest_supervisor_archive_path}" --forward-port "${host_port}:${guest_port}" ) diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index 0f64247731..c700801f6f 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -337,6 +337,31 @@ resolve_docker_supervisor_image() { printf '%s\n' "openshell/supervisor:dev" } +resolve_docker_sandbox_runtime_image() { + if [ -n "${OPENSHELL_DOCKER_SANDBOX_RUNTIME_IMAGE:-}" ]; then + printf '%s\n' "${OPENSHELL_DOCKER_SANDBOX_RUNTIME_IMAGE}" + return 0 + fi + + if [ -n "${OPENSHELL_SANDBOX_RUNTIME_IMAGE:-}" ]; then + printf '%s\n' "${OPENSHELL_SANDBOX_RUNTIME_IMAGE}" + return 0 + fi + + if [ -n "${CI:-}" ]; then + if [ -z "${IMAGE_TAG:-}" ]; then + echo "ERROR: IMAGE_TAG must be set in CI when no Docker sandbox runtime image override is provided." >&2 + exit 2 + fi + + local registry="${OPENSHELL_REGISTRY:-ghcr.io/nvidia/openshell}" + printf '%s/sandbox:%s\n' "${registry%/}" "${IMAGE_TAG}" + return 0 + fi + + printf '%s\n' "openshell/sandbox:dev" +} + docker_pull_with_retry() { local image=$1 local attempts=4 @@ -384,6 +409,27 @@ build_local_docker_supervisor_image_if_required() { exit 2 } +build_local_docker_sandbox_runtime_image_if_required() { + local image=$1 + + if [ "${image}" != "openshell/sandbox:dev" ]; then + return 0 + fi + + local daemon_arch + daemon_arch="$(ce_info_arch)" + + echo "Building local Docker sandbox runtime image ${image} for linux/${daemon_arch}..." + CONTAINER_ENGINE=docker DOCKER_PLATFORM="linux/${daemon_arch}" IMAGE_TAG=dev \ + bash "${ROOT}/tasks/scripts/docker-build-image.sh" sandbox + if docker image inspect "${image}" >/dev/null 2>&1; then + return 0 + fi + + echo "ERROR: expected sandbox runtime image '${image}' after local build." >&2 + exit 2 +} + ensure_docker_supervisor_image() { local image=$1 @@ -401,6 +447,23 @@ ensure_docker_supervisor_image() { exit 2 } +ensure_docker_sandbox_runtime_image() { + local image=$1 + + if docker image inspect "${image}" >/dev/null 2>&1; then + return 0 + fi + + echo "Pulling Docker sandbox runtime image ${image}..." + if docker_pull_with_retry "${image}"; then + return 0 + fi + + echo "ERROR: sandbox runtime image '${image}' is not available." >&2 + echo " Build it, push it, or set OPENSHELL_SANDBOX_RUNTIME_IMAGE to a pullable image." >&2 + exit 2 +} + image_uses_latest_tag() { local image=$1 local last_component @@ -448,6 +511,11 @@ build_local_docker_supervisor_image_if_required "${SUPERVISOR_IMAGE}" ensure_docker_supervisor_image "${SUPERVISOR_IMAGE}" echo "Using Docker supervisor image: ${SUPERVISOR_IMAGE}" +SANDBOX_RUNTIME_IMAGE="$(resolve_docker_sandbox_runtime_image)" +build_local_docker_sandbox_runtime_image_if_required "${SANDBOX_RUNTIME_IMAGE}" +ensure_docker_sandbox_runtime_image "${SANDBOX_RUNTIME_IMAGE}" +echo "Using Docker sandbox runtime image: ${SANDBOX_RUNTIME_IMAGE}" + DEFAULT_SANDBOX_IMAGE="ghcr.io/nvidia/openshell-community/sandboxes/base:latest" SANDBOX_IMAGE="${OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE:-${OPENSHELL_SANDBOX_IMAGE:-${DEFAULT_SANDBOX_IMAGE}}}" SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE_PULL_POLICY:-${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}}" @@ -523,6 +591,7 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" printf 'image_pull_policy = %s\n' "$(toml_string "${SANDBOX_IMAGE_PULL_POLICY}")" printf 'enable_bind_mounts = true\n' + printf 'sandbox_runtime_image = %s\n' "$(toml_string "${SANDBOX_RUNTIME_IMAGE}")" printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" if [ -n "${GATEWAY_HOST_ALIAS_IP}" ]; then printf 'host_gateway_ip = %s\n' "$(toml_string "${GATEWAY_HOST_ALIAS_IP}")" @@ -541,6 +610,7 @@ if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" printf 'guest_tls_key = %s\n' "$(toml_string "${PKI_DIR}/client/tls.key")" printf 'enable_bind_mounts = true\n' + printf 'sandbox_runtime_image = %s\n' "$(toml_string "${SANDBOX_RUNTIME_IMAGE}")" printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" if [ -n "${GATEWAY_HOST_ALIAS_IP}" ]; then printf 'host_gateway_ip = %s\n' "$(toml_string "${GATEWAY_HOST_ALIAS_IP}")" From ebd2cd27c9c06bf6a64834d60f65269fb9aa61f4 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 11 Sep 2026 15:02:07 -0700 Subject: [PATCH 07/19] fix(docker): restore startup validation after rebase Signed-off-by: Drew Newberry --- Cargo.lock | 1 - crates/openshell-driver-docker/src/lib.rs | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index ce581c6bc3..c35f5f0516 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7278,7 +7278,6 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96374855068f47402c3121c6eed88d29cb1de8f3ab27090e273e420bdabcf050" dependencies = [ - "futures", "parking_lot", ] diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index b41ac6d2b1..8db33a1c38 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -201,6 +201,28 @@ pub struct DockerComputeConfig { pub enable_bind_mounts: bool, } +impl DockerComputeConfig { + /// Validate startup configuration without connecting to Docker. + pub fn validate_configuration(&self, gateway_bind_address: SocketAddr) -> CoreResult<()> { + if let Some(socket_path) = self.socket_path.as_deref() + && socket_path.to_str().is_none() + { + return Err(Error::config(format!( + "Docker socket path is not valid UTF-8: {}", + socket_path.display() + ))); + } + validate_sandbox_pids_limit(self.sandbox_pids_limit)?; + parse_optional_host_gateway_ip(&self.host_gateway_ip)?; + if gateway_bind_address.port() == 0 { + return Err(Error::config( + "docker compute driver requires a fixed non-zero gateway bind port", + )); + } + Ok(()) + } +} + impl Default for DockerComputeConfig { fn default() -> Self { Self { From 5857b3fb05ae0ea8d7c4a05de014f987d185ce8c Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 12 Sep 2026 13:54:28 -0700 Subject: [PATCH 08/19] refactor(docker): name the sandbox runtime directly Signed-off-by: Drew Newberry --- crates/openshell-driver-docker/README.md | 2 +- .../openshell-driver-docker/src/isolation.rs | 18 +-- crates/openshell-driver-docker/src/lib.rs | 113 +++++++++--------- crates/openshell-driver-docker/src/tests.rs | 4 +- 4 files changed, 71 insertions(+), 66 deletions(-) diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 803da311c4..6281e22030 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -86,7 +86,7 @@ and creates a new supervisor companion. A durably stopped sandbox stays stopped across gateway restarts. Delete force-removes both containers, the driver-owned channel volume, and the -host-private topology record. Missing or altered topology and channel resources +host-private runtime descriptor. Missing or altered descriptor and channel resources fail closed; the driver does not run an older combined-supervisor layout. ## Driver Config Mounts diff --git a/crates/openshell-driver-docker/src/isolation.rs b/crates/openshell-driver-docker/src/isolation.rs index 4dff02789b..cceec1c9f6 100644 --- a/crates/openshell-driver-docker/src/isolation.rs +++ b/crates/openshell-driver-docker/src/isolation.rs @@ -3,7 +3,7 @@ //! Docker provisioning for the shared authenticated boundary protocol. //! -//! Docker owns only the container/socket topology and immutable OCI resource +//! Docker owns only container placement, the protected socket, and immutable OCI resource //! claims. Lifecycle, process, network, identity, and wire behavior live in //! `openshell-isolation-interface` and `openshell-sandbox`. @@ -13,7 +13,7 @@ use std::path::PathBuf; use openshell_isolation_interface::contract::{DriverFenceEvidence, ResolvedWorkloadIdentity}; use openshell_sandbox_backend::boundary_protocol::{ - BoundaryConfig, BoundaryListener, BoundaryTopology, GatewayVerificationKey, + BoundaryConfig, BoundaryListener, GatewayVerificationKey, SandboxRuntimeDescriptor, SandboxTlsClientConfig, SandboxTlsServerConfig, SandboxTransport, }; @@ -38,7 +38,7 @@ pub struct DockerBoundarySpec { /// Protected container config and matching host descriptor. pub struct DockerBoundaryProvisioning { pub boundary_config: BoundaryConfig, - pub topology: BoundaryTopology, + pub runtime_descriptor: SandboxRuntimeDescriptor, } impl DockerBoundarySpec { @@ -72,7 +72,7 @@ impl DockerBoundarySpec { driver_fence: driver_fence.clone(), child_env: self.child_env, }, - topology: BoundaryTopology { + runtime_descriptor: SandboxRuntimeDescriptor { boundary_id: self.boundary_id, generation: self.generation, session_id: self.session_id, @@ -135,21 +135,21 @@ mod tests { assert_eq!( provisioned.boundary_config.resource_claims, - provisioned.topology.resource_claims + provisioned.runtime_descriptor.resource_claims ); assert_eq!( - provisioned.topology.resource_claims["docker.container_id"], + provisioned.runtime_descriptor.resource_claims["docker.container_id"], "sha256:container" ); assert_eq!( provisioned.boundary_config.driver_fence, - provisioned.topology.driver_fence + provisioned.runtime_descriptor.driver_fence ); assert!( provisioned - .topology + .runtime_descriptor .driver_fence - .validate_for_backend("docker") + .validate() .is_ok() ); } diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 8db33a1c38..2efaadb2f2 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -26,9 +26,9 @@ use futures::{Stream, StreamExt}; use openshell_core::config::{DEFAULT_SANDBOX_PIDS_LIMIT, DEFAULT_STOP_TIMEOUT_SECS}; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ - CONDITION_EXITED, CONDITION_RUNTIME_RESTART, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, - LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, - GatewayCallbackRoute, SANDBOX_RUNTIME_IMAGE_BINARY_PATH, extract_first_tar_entry, + CONDITION_EXITED, CONDITION_RUNTIME_RESTART, GatewayCallbackRoute, LABEL_MANAGED_BY, + LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, + LABEL_SANDBOX_WORKSPACE, SANDBOX_RUNTIME_IMAGE_BINARY_PATH, extract_first_tar_entry, gateway_callback_endpoint, supervisor_image_should_refresh, temp_extract_container_name, }; use openshell_core::gpu::{ @@ -60,7 +60,7 @@ use openshell_core::proto_struct::{ use openshell_core::{Error, Result as CoreResult}; use openshell_isolation_interface::contract::ResolvedWorkloadIdentity; use openshell_sandbox_backend::boundary_protocol::{ - BoundaryConfig, BoundaryTopology, GatewayVerificationKey, SandboxTlsClientConfig, + BoundaryConfig, GatewayVerificationKey, SandboxRuntimeDescriptor, SandboxTlsClientConfig, SandboxTlsServerConfig, generate_sandbox_tls_material, }; use opentelemetry::trace::TraceContextExt as _; @@ -106,16 +106,16 @@ const BOUNDARY_SOCKET_MOUNT_PATH: &str = "/.openshell/channel/sandbox/control.so const BOUNDARY_CERTIFICATE_MOUNT_PATH: &str = "/.openshell/channel/sandbox/server.crt"; const BOUNDARY_PRIVATE_KEY_MOUNT_PATH: &str = "/.openshell/channel/sandbox/server.key"; const SUPERVISOR_STATE_MOUNT_PATH: &str = "/.openshell/channel/supervisor"; -const DRIVER_ADMITTED_BACKEND: &str = "docker"; -const LABEL_ISOLATION_TOPOLOGY: &str = "openshell.ai/isolation-topology"; -const LABEL_ISOLATION_TOPOLOGY_CAPABILITY_FREE: &str = "capability-free"; +const DRIVER_ADMITTED_BACKEND: &str = openshell_sandbox_backend::BACKEND_NAME; +const LABEL_ISOLATION_BACKEND: &str = "openshell.ai/isolation-backend"; +const LABEL_ISOLATION_BACKEND_OPEN_SHELL: &str = openshell_sandbox_backend::BACKEND_NAME; const LABEL_ISOLATION_ROLE: &str = "openshell.ai/isolation-role"; const LABEL_ISOLATION_ROLE_SANDBOX: &str = "sandbox"; const LABEL_ISOLATION_ROLE_SUPERVISOR: &str = "supervisor"; const SUPERVISOR_NETWORK_MODE: &str = "host"; const LABEL_ISOLATION_ROLE_STAGING: &str = "staging"; const LABEL_ISOLATION_ROLE_IDENTITY: &str = "identity"; -const TOPOLOGY_PAYLOAD_FILE: &str = "topology.payload"; +const RUNTIME_DESCRIPTOR_FILE: &str = "runtime-descriptor.json"; const MAIN_PROCESS_SPEC_FILE: &str = "main-process.json"; const WORKSPACE_ROOT_FILE: &str = "workspace-root"; const BOUNDARY_CONFIG_FILE: &str = "boundary-bootstrap.json"; @@ -1542,7 +1542,7 @@ impl DockerComputeDriver { HashMap::from([("container_name".to_string(), container_name.clone())]), ); - let topology = match prepare_docker_boundary_files( + match prepare_docker_boundary_files( &self.docker, sandbox, &self.config, @@ -1552,7 +1552,7 @@ impl DockerComputeDriver { ) .await { - Ok(topology) => topology, + Ok(()) => {} Err(status) => { let _ = self .docker @@ -1569,7 +1569,7 @@ impl DockerComputeDriver { status.message(), )); } - }; + } let start_result = async { openshell_otel::record_error_result( @@ -1614,7 +1614,6 @@ impl DockerComputeDriver { &self.docker, sandbox, &self.config, - &topology, failure_context, ) .await @@ -1845,7 +1844,7 @@ impl DockerComputeDriver { "{LABEL_SANDBOX_NAMESPACE}={}", self.config.sandbox_namespace ), - format!("{LABEL_ISOLATION_TOPOLOGY}={LABEL_ISOLATION_TOPOLOGY_CAPABILITY_FREE}"), + format!("{LABEL_ISOLATION_BACKEND}={LABEL_ISOLATION_BACKEND_OPEN_SHELL}"), ]); let volumes = self .docker @@ -1915,12 +1914,15 @@ impl DockerComputeDriver { if let Some(stale) = stale { stop_docker_control_process(stale).await; } - let Some(topology) = read_docker_boundary_topology(&sandbox.id, &self.config).await? else { + if read_docker_runtime_descriptor(&sandbox.id, &self.config) + .await? + .is_none() + { let container_id = summary_container_target(container) .ok_or_else(|| Status::internal("managed Docker container has no id or name"))?; let failure_context = self.control_failure_context(sandbox.clone(), container_id); let status = Status::failed_precondition( - "Docker sandbox topology is missing; refusing to leave the workload running without its supervisor", + "Docker sandbox runtime descriptor is missing; refusing to leave the workload running without its supervisor", ); handle_docker_runtime_failure( failure_context, @@ -1929,7 +1931,7 @@ impl DockerComputeDriver { ) .await; return Err(status); - }; + } let container_id = summary_container_target(container) .ok_or_else(|| Status::internal("managed Docker container has no id or name"))?; self.clear_runtime_failure(&sandbox.id).await; @@ -1938,7 +1940,6 @@ impl DockerComputeDriver { &self.docker, &sandbox, &self.config, - &topology, failure_context.clone(), ) .await @@ -2223,11 +2224,11 @@ impl DockerComputeDriver { launch_authentication, ) .await?; - let Some(topology) = - read_docker_boundary_topology(resolved_sandbox_id, &self.config).await? + let Some(runtime_descriptor) = + read_docker_runtime_descriptor(resolved_sandbox_id, &self.config).await? else { return Err(Status::failed_precondition( - "Docker sandbox topology is missing; refusing to start the workload without its supervisor", + "Docker sandbox runtime descriptor is missing; refusing to start the workload without its supervisor", )); }; let boundary_config = tokio::fs::read( @@ -2272,7 +2273,7 @@ impl DockerComputeDriver { &self.docker, &target, &self.config, - &topology.workload_identity, + &runtime_descriptor.workload_identity, &boundary_config, DockerSandboxTls { certificate: &boundary_certificate, @@ -3787,8 +3788,8 @@ async fn create_docker_channel_volume( config.sandbox_namespace.clone(), ), ( - LABEL_ISOLATION_TOPOLOGY.to_string(), - LABEL_ISOLATION_TOPOLOGY_CAPABILITY_FREE.to_string(), + LABEL_ISOLATION_BACKEND.to_string(), + LABEL_ISOLATION_BACKEND_OPEN_SHELL.to_string(), ), ]); docker @@ -4122,7 +4123,7 @@ async fn prepare_docker_boundary_files( container_id: &str, image: &DockerImageMetadata, workload_identity: &ResolvedWorkloadIdentity, -) -> Result { +) -> Result<(), Status> { let directory = docker_boundary_state_dir(sandbox, config)?; let workspace_root = driver_mounts::resolve_oci_workspace_root(&image.working_dir) .map_err(Status::failed_precondition)?; @@ -4191,10 +4192,14 @@ async fn prepare_docker_boundary_files( ) .await?; let descriptor = provisioning - .topology - .descriptor(DRIVER_ADMITTED_BACKEND) + .runtime_descriptor + .backend_descriptor() .map_err(|error| Status::internal(error.to_string()))?; - write_docker_boundary_file(&directory.join(TOPOLOGY_PAYLOAD_FILE), &descriptor.payload).await?; + write_docker_boundary_file( + &directory.join(RUNTIME_DESCRIPTOR_FILE), + &descriptor.payload, + ) + .await?; let supervisor_auth = serde_json::to_vec(&launch_authentication.supervisor) .map_err(|error| Status::internal(format!("encode Docker supervisor auth: {error}")))?; write_docker_boundary_file( @@ -4216,7 +4221,7 @@ async fn prepare_docker_boundary_files( workspace_root.as_bytes(), ) .await?; - Ok(provisioning.topology) + Ok(()) } async fn docker_supervisor_bundle_archive( @@ -4224,9 +4229,9 @@ async fn docker_supervisor_bundle_archive( config: &DockerDriverRuntimeConfig, ) -> Result, Status> { let directory = docker_boundary_state_dir(sandbox, config)?; - let topology = tokio::fs::read(directory.join(TOPOLOGY_PAYLOAD_FILE)) + let runtime_descriptor = tokio::fs::read(directory.join(RUNTIME_DESCRIPTOR_FILE)) .await - .map_err(|error| Status::internal(format!("read Docker topology payload: {error}")))?; + .map_err(|error| Status::internal(format!("read Docker runtime descriptor: {error}")))?; let auth_bundle = tokio::fs::read(directory.join(SUPERVISOR_AUTH_BUNDLE_FILE)) .await .map_err(|error| { @@ -4247,11 +4252,11 @@ async fn docker_supervisor_bundle_archive( )?; append_docker_archive_file( &mut archive, - ".openshell/channel/supervisor/topology.payload", + ".openshell/channel/supervisor/runtime-descriptor.json", 0o600, SUPERVISOR_UID, SUPERVISOR_GID, - &topology, + &runtime_descriptor, )?; append_docker_archive_file( &mut archive, @@ -4316,9 +4321,10 @@ async fn refresh_docker_boundary_authentication( "decode Docker sandbox bootstrap for authentication rotation: {error}" )) })?; - let Some(mut topology) = read_docker_boundary_topology(sandbox_id, config).await? else { + let Some(mut runtime_descriptor) = read_docker_runtime_descriptor(sandbox_id, config).await? + else { return Err(Status::failed_precondition( - "Docker sandbox topology is missing during authentication rotation", + "Docker sandbox runtime descriptor is missing during authentication rotation", )); }; let session_id = authentication.supervisor.session_id; @@ -4328,16 +4334,16 @@ async fn refresh_docker_boundary_authentication( boundary_config.gateway_id = authentication.gateway_id; boundary_config.verification_keys = gateway_verification_keys(&authentication.verification_keys)?; - topology.session_id = session_id; - topology.tls = SandboxTlsClientConfig { + runtime_descriptor.session_id = session_id; + runtime_descriptor.tls = SandboxTlsClientConfig { server_name: tls.server_name, trust_anchor_pem: tls.trust_anchor_pem, }; let encoded_boundary_config = boundary_config .encode() .map_err(|error| Status::internal(error.to_string()))?; - let descriptor = topology - .descriptor(DRIVER_ADMITTED_BACKEND) + let descriptor = runtime_descriptor + .backend_descriptor() .map_err(|error| Status::internal(error.to_string()))?; let supervisor_auth = serde_json::to_vec(&authentication.supervisor) .map_err(|error| Status::internal(format!("encode Docker supervisor auth: {error}")))?; @@ -4356,7 +4362,11 @@ async fn refresh_docker_boundary_authentication( tls.private_key_pem.as_bytes(), ) .await?; - write_docker_boundary_file(&directory.join(TOPOLOGY_PAYLOAD_FILE), &descriptor.payload).await?; + write_docker_boundary_file( + &directory.join(RUNTIME_DESCRIPTOR_FILE), + &descriptor.payload, + ) + .await?; write_docker_boundary_file( &directory.join(SUPERVISOR_AUTH_BUNDLE_FILE), &supervisor_auth, @@ -4364,24 +4374,24 @@ async fn refresh_docker_boundary_authentication( .await } -async fn read_docker_boundary_topology( +async fn read_docker_runtime_descriptor( sandbox_id: &str, config: &DockerDriverRuntimeConfig, -) -> Result, Status> { - let path = docker_boundary_state_dir_by_id(sandbox_id, config)?.join(TOPOLOGY_PAYLOAD_FILE); +) -> Result, Status> { + let path = docker_boundary_state_dir_by_id(sandbox_id, config)?.join(RUNTIME_DESCRIPTOR_FILE); let bytes = match tokio::fs::read(&path).await { Ok(bytes) => bytes, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(error) => { return Err(Status::internal(format!( - "read Docker boundary topology {}: {error}", + "read Docker runtime descriptor {}: {error}", path.display() ))); } }; serde_json::from_slice(&bytes).map(Some).map_err(|error| { Status::internal(format!( - "decode Docker boundary topology {}: {error}", + "decode Docker runtime descriptor {}: {error}", path.display() )) }) @@ -4492,13 +4502,9 @@ async fn spawn_docker_control_process( docker: &Docker, sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, - topology: &BoundaryTopology, failure_context: DockerRuntimeFailureContext, ) -> Result { let directory = docker_boundary_state_dir(sandbox, config)?; - let descriptor = topology - .descriptor(DRIVER_ADMITTED_BACKEND) - .map_err(|error| Status::internal(error.to_string()))?; let main_process_spec = tokio::fs::read_to_string(directory.join(MAIN_PROCESS_SPEC_FILE)) .await .map_err(|error| Status::internal(format!("read Docker main process spec: {error}")))?; @@ -4512,7 +4518,7 @@ async fn spawn_docker_control_process( Some(RemoveContainerOptionsBuilder::default().force(true).build()), ) .await; - let topology_path = format!("{SUPERVISOR_STATE_MOUNT_PATH}/topology.payload"); + let runtime_descriptor_path = format!("{SUPERVISOR_STATE_MOUNT_PATH}/runtime-descriptor.json"); let auth_bundle_path = format!("{SUPERVISOR_STATE_MOUNT_PATH}/auth.json"); let mut environment = vec![ format!( @@ -4596,9 +4602,8 @@ async fn spawn_docker_control_process( user: Some(format!("{SUPERVISOR_UID}:{SUPERVISOR_GID}")), entrypoint: Some(vec![SUPERVISOR_IMAGE_CONTROL_BINARY_PATH.to_string()]), cmd: Some(vec![ - format!("--topology-backend-name={}", descriptor.backend_name), - "--topology-payload-file".to_string(), - topology_path.clone(), + "--backend-descriptor-file".to_string(), + runtime_descriptor_path, "--auth-bundle-file".to_string(), auth_bundle_path, "--workdir".to_string(), @@ -5214,8 +5219,8 @@ fn build_container_create_body_for_image( config.sandbox_namespace.clone(), ); labels.insert( - LABEL_ISOLATION_TOPOLOGY.to_string(), - LABEL_ISOLATION_TOPOLOGY_CAPABILITY_FREE.to_string(), + LABEL_ISOLATION_BACKEND.to_string(), + LABEL_ISOLATION_BACKEND_OPEN_SHELL.to_string(), ); labels.insert( LABEL_ISOLATION_ROLE.to_string(), diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index f34015c4aa..d63b04bfd7 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -1387,9 +1387,9 @@ fn container_creation_uses_inspected_immutable_image() { assert_eq!( body.labels .as_ref() - .and_then(|labels| labels.get(LABEL_ISOLATION_TOPOLOGY)) + .and_then(|labels| labels.get(LABEL_ISOLATION_BACKEND)) .map(String::as_str), - Some(LABEL_ISOLATION_TOPOLOGY_CAPABILITY_FREE) + Some(LABEL_ISOLATION_BACKEND_OPEN_SHELL) ); assert_eq!( body.labels From ecab00b4c372bd9c3fcf06b51da371c6f4e53b5b Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 12 Sep 2026 23:06:06 -0700 Subject: [PATCH 09/19] fix(docker): narrow supervisor CA runtime storage Signed-off-by: Drew Newberry --- crates/openshell-driver-docker/README.md | 2 +- crates/openshell-driver-docker/src/lib.rs | 4 ++-- crates/openshell-driver-docker/src/tests.rs | 14 ++++++++++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 6281e22030..cb022705fb 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -74,7 +74,7 @@ LSM decisions remain authoritative. | `restart_policy = no` | Keeps canonical main-process exit terminal. | | `PidsLimit` | Applies the configured sandbox PID budget. Set `sandbox_pids_limit = 0` to use the runtime default. | | Private named volume | Carries a per-generation mutual-TLS sandbox/supervisor channel without sharing daemon-host paths. The sandbox consumes its server key at startup; only the supervisor receives the client key. | -| In-memory `/run` tmpfs | Supplies writable runtime state without changing the workload image root filesystem. | +| In-memory `/run/openshell-supervisor-ca` tmpfs | Holds only the public supervisor CA certificate and trust bundle without making all of `/run` writable. | | CDI GPU request | Assigns the exact validated CDI devices requested by driver config or count-based selection. | ## Stop, Start, and Delete diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 2efaadb2f2..e0c95d5ad6 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -5271,9 +5271,9 @@ fn build_container_create_body_for_image( network_mode: Some("none".to_string()), dns: Some(vec!["127.0.0.53".to_string()]), tmpfs: Some(HashMap::from([( - "/run".to_string(), + openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_DIR.to_string(), format!( - "rw,noexec,nosuid,size=64m,uid={},gid={},mode=0755", + "rw,noexec,nosuid,nodev,size=1m,uid={},gid={},mode=0755", workload_identity.uid, workload_identity.gid ), )])), diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index d63b04bfd7..41cd26c577 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -2828,6 +2828,20 @@ fn build_container_create_body_disables_docker_networking() { assert_eq!(host_config.dns, Some(vec!["127.0.0.53".to_string()])); } +#[test] +fn build_container_create_body_limits_writable_runtime_storage_to_supervisor_ca() { + let create_body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); + let host_config = create_body.host_config.expect("host_config is populated"); + let tmpfs = host_config.tmpfs.expect("sandbox tmpfs is populated"); + + assert_eq!(tmpfs.len(), 1); + assert_eq!( + tmpfs.get(openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_DIR), + Some(&"rw,noexec,nosuid,nodev,size=1m,uid=1000,gid=1000,mode=0755".to_string()) + ); + assert!(!tmpfs.contains_key("/run")); +} + #[test] fn build_container_create_body_uses_runtime_namespace_label() { // Regression test: the namespace label must come from the driver's From 9d25f436cee0788233758344fba28c93ccdcb06e Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 13 Sep 2026 00:24:30 -0700 Subject: [PATCH 10/19] fix(docker): close companion isolation gaps Signed-off-by: Drew Newberry --- crates/openshell-driver-docker/README.md | 13 +- crates/openshell-driver-docker/src/lib.rs | 490 +++++++++++++++----- crates/openshell-driver-docker/src/tests.rs | 128 ++++- docs/reference/gateway-config.mdx | 12 +- docs/reference/sandbox-compute-drivers.mdx | 2 +- e2e/python/test_sandbox_policy.py | 94 +++- e2e/run.sh | 24 +- 7 files changed, 612 insertions(+), 151 deletions(-) diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index cb022705fb..b44ffb4b37 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -25,9 +25,10 @@ The driver creates two containers for each sandbox: inspection, DNS policy, and external upstream connections. Both containers are non-root, request no capabilities, and set -no-new-privileges. They share only a driver-created Docker named volume. The -volume carries an authenticated Unix socket and immutable bootstrap material; -it is writable by the sandbox and read-only in the supervisor. +no-new-privileges. A shared named volume carries the authenticated Unix socket +and sandbox bootstrap material. A second supervisor-only volume carries the +supervisor JWT and gateway client credentials and is never mounted into the +workload. The workload uses `network_mode=none`. Its seccomp user-notification broker mediates every supported TCP and DNS operation, attributes it to the calling @@ -72,8 +73,8 @@ LSM decisions remain authoritative. | `network_mode = none` on the workload | Removes direct external routes. | | `network_mode = host` on the supervisor | Lets the trusted supervisor originate approved gateway and upstream connections through the daemon host network. | | `restart_policy = no` | Keeps canonical main-process exit terminal. | -| `PidsLimit` | Applies the configured sandbox PID budget. Set `sandbox_pids_limit = 0` to use the runtime default. | -| Private named volume | Carries a per-generation mutual-TLS sandbox/supervisor channel without sharing daemon-host paths. The sandbox consumes its server key at startup; only the supervisor receives the client key. | +| `PidsLimit` | Applies the configured sandbox PID budget. Omit `sandbox_pids_limit` to use OpenShell's default. Explicit zero is invalid. | +| Private named volumes | One carries the authenticated sandbox/supervisor channel. The other is mounted only into the supervisor and contains its JWT and private gateway credentials. | | In-memory `/run/openshell-supervisor-ca` tmpfs | Holds only the public supervisor CA certificate and trust bundle without making all of `/run` writable. | | CDI GPU request | Assigns the exact validated CDI devices requested by driver config or count-based selection. | @@ -85,7 +86,7 @@ volumes. Start stages a fresh sandbox bootstrap bundle, restarts that workload, and creates a new supervisor companion. A durably stopped sandbox stays stopped across gateway restarts. -Delete force-removes both containers, the driver-owned channel volume, and the +Delete force-removes both containers, the driver-owned runtime volumes, and the host-private runtime descriptor. Missing or altered descriptor and channel resources fail closed; the driver does not run an older combined-supervisor layout. diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index e0c95d5ad6..959e8da8cf 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -23,13 +23,14 @@ use bollard::query_parameters::{ }; use bytes::Bytes; use futures::{Stream, StreamExt}; -use openshell_core::config::{DEFAULT_SANDBOX_PIDS_LIMIT, DEFAULT_STOP_TIMEOUT_SECS}; +use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ CONDITION_EXITED, CONDITION_RUNTIME_RESTART, GatewayCallbackRoute, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, SANDBOX_RUNTIME_IMAGE_BINARY_PATH, extract_first_tar_entry, gateway_callback_endpoint, supervisor_image_should_refresh, temp_extract_container_name, + validate_linux_elf_binary, }; use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, @@ -57,7 +58,9 @@ use openshell_core::proto::compute::v1::{ use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, }; -use openshell_core::{Error, Result as CoreResult}; +use openshell_core::{ + AppArmorProfile, Error, ImagePullPolicy, Result as CoreResult, UpstreamProxyConfig, +}; use openshell_isolation_interface::contract::ResolvedWorkloadIdentity; use openshell_sandbox_backend::boundary_protocol::{ BoundaryConfig, GatewayVerificationKey, SandboxRuntimeDescriptor, SandboxTlsClientConfig, @@ -105,7 +108,10 @@ const BOUNDARY_CONFIG_MOUNT_PATH: &str = "/.openshell/channel/sandbox/bootstrap. const BOUNDARY_SOCKET_MOUNT_PATH: &str = "/.openshell/channel/sandbox/control.sock"; const BOUNDARY_CERTIFICATE_MOUNT_PATH: &str = "/.openshell/channel/sandbox/server.crt"; const BOUNDARY_PRIVATE_KEY_MOUNT_PATH: &str = "/.openshell/channel/sandbox/server.key"; -const SUPERVISOR_STATE_MOUNT_PATH: &str = "/.openshell/channel/supervisor"; +const SUPERVISOR_STATE_MOUNT_PATH: &str = "/.openshell/supervisor"; +const SUPERVISOR_PROXY_AUTH_MOUNT_PATH: &str = "/.openshell/supervisor/upstream-proxy-auth"; +const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str = + openshell_core::driver_utils::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR; const DRIVER_ADMITTED_BACKEND: &str = openshell_sandbox_backend::BACKEND_NAME; const LABEL_ISOLATION_BACKEND: &str = "openshell.ai/isolation-backend"; const LABEL_ISOLATION_BACKEND_OPEN_SHELL: &str = openshell_sandbox_backend::BACKEND_NAME; @@ -161,10 +167,10 @@ pub struct DockerComputeConfig { pub default_image: String, /// Image pull policy for sandbox images. - pub image_pull_policy: String, + pub image_pull_policy: ImagePullPolicy, - /// Namespace label applied to Docker sandboxes. - pub sandbox_namespace: String, + /// Value of the `openshell.sandbox_namespace` label applied to Docker sandboxes. + pub sandbox_label: String, /// Gateway gRPC endpoint the sandbox connects back to. pub grpc_endpoint: String, @@ -172,6 +178,12 @@ pub struct DockerComputeConfig { /// Image containing the trusted `openshell-sandbox` binary. pub sandbox_runtime_image: Option, + /// Optional host path to the trusted `openshell-sandbox` binary. + /// + /// This preserves the original `supervisor_bin` configuration name while + /// the split runtime transitions to the explicit sandbox image setting. + pub supervisor_bin: Option, + /// Image containing the trusted `openshell-supervisor` binary. pub supervisor_image: Option, @@ -190,15 +202,34 @@ pub struct DockerComputeConfig { /// Host gateway IP used for sandbox host aliases. pub host_gateway_ip: String, + /// Unix socket path used for interactive sandbox access. + pub ssh_socket_path: String, + /// Container cgroup PID limit for Docker-managed sandboxes. /// - /// Set to `0` to leave Docker's runtime/default PID limit unchanged. - pub sandbox_pids_limit: i64, + /// Omit the field to use `OpenShell`'s default sandbox process limit. + /// Explicit zero is invalid. + #[serde( + default = "openshell_core::config::default_sandbox_pids_limit", + skip_serializing_if = "Option::is_none" + )] + pub sandbox_pids_limit: Option, /// Allow sandbox requests to attach host bind mounts through /// `template.driver_config`. #[serde(default)] pub enable_bind_mounts: bool, + + /// Corporate forward-proxy settings supplied to the supervisor. + #[serde(flatten)] + pub upstream_proxy: UpstreamProxyConfig, + + /// Host UNIX socket projected into the supervisor for provider identity. + pub provider_spiffe_workload_api_socket: Option, + + /// `AppArmor` confinement requested for the workload container. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub app_armor_profile: Option, } impl DockerComputeConfig { @@ -213,6 +244,13 @@ impl DockerComputeConfig { ))); } validate_sandbox_pids_limit(self.sandbox_pids_limit)?; + validate_image_pull_policy(self.image_pull_policy)?; + self.upstream_proxy.validate().map_err(Error::config)?; + validate_docker_proxy_auth_file(&self.upstream_proxy)?; + if let Some(socket) = self.provider_spiffe_workload_api_socket.as_deref() { + openshell_core::driver_utils::validate_provider_spiffe_unix_socket(socket) + .map_err(Error::config)?; + } parse_optional_host_gateway_ip(&self.host_gateway_ip)?; if gateway_bind_address.port() == 0 { return Err(Error::config( @@ -228,18 +266,23 @@ impl Default for DockerComputeConfig { Self { socket_path: None, default_image: openshell_core::image::default_sandbox_image(), - image_pull_policy: String::new(), - sandbox_namespace: "default".to_string(), + image_pull_policy: ImagePullPolicy::default(), + sandbox_label: "default".to_string(), grpc_endpoint: String::new(), sandbox_runtime_image: None, + supervisor_bin: None, supervisor_image: None, guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), host_gateway_ip: String::new(), - sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, + ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), + sandbox_pids_limit: openshell_core::config::default_sandbox_pids_limit(), enable_bind_mounts: false, + upstream_proxy: UpstreamProxyConfig::default(), + provider_spiffe_workload_api_socket: None, + app_armor_profile: None, } } } @@ -254,7 +297,7 @@ pub(crate) struct DockerGuestTlsPaths { #[derive(Debug, Clone)] struct DockerDriverRuntimeConfig { default_image: String, - image_pull_policy: String, + image_pull_policy: ImagePullPolicy, sandbox_namespace: String, gateway_route: DockerGatewayRoute, gateway_callback_bind_address: Option, @@ -264,11 +307,15 @@ struct DockerDriverRuntimeConfig { supervisor_image_id: String, supervisor_grpc_endpoint: String, gateway_tls_server_name: Option, + ssh_socket_path: String, guest_tls: Option, daemon_version: String, gpu: DockerGpuRuntimeCapabilities, - sandbox_pids_limit: i64, + sandbox_pids_limit: Option, enable_bind_mounts: bool, + upstream_proxy: UpstreamProxyConfig, + provider_spiffe_workload_api_socket: Option, + app_armor_profile: Option, } #[derive(Debug, Clone, Copy)] @@ -827,6 +874,8 @@ impl DockerComputeDriver { wsl_all_gpu_fallback_enabled, }; validate_sandbox_pids_limit(docker_config.sandbox_pids_limit)?; + validate_image_pull_policy(docker_config.image_pull_policy)?; + validate_docker_app_armor_profile(docker_config.app_armor_profile.as_ref(), &info)?; let gateway_port = gateway_bind_address.port(); if gateway_port == 0 { return Err(Error::config( @@ -879,28 +928,38 @@ impl DockerComputeDriver { .unwrap_or_else(openshell_core::config::default_supervisor_image); let supervisor_image_id = ensure_runtime_image(&docker, &supervisor_image, "supervisor").await?; - let sandbox_runtime_image = docker_config - .sandbox_runtime_image - .clone() - .unwrap_or_else(openshell_core::config::default_sandbox_runtime_image); - let sandbox_runtime_image_id = - ensure_runtime_image(&docker, &sandbox_runtime_image, "sandbox runtime").await?; - let sandbox_binary = Arc::new( - extract_sandbox_binary_bytes(&docker, &sandbox_runtime_image_id) - .await - .map_err(|error| { - Error::config(format!( - "failed to load trusted sandbox binary from Docker image '{sandbox_runtime_image}': {error}" - )) - })?, - ); + let sandbox_binary = if let Some(path) = docker_config.supervisor_bin.as_deref() { + validate_linux_elf_binary(path).map_err(Error::config)?; + Arc::new(tokio::fs::read(path).await.map_err(|error| { + Error::config(format!( + "failed to read trusted sandbox binary '{}': {error}", + path.display() + )) + })?) + } else { + let sandbox_runtime_image = docker_config + .sandbox_runtime_image + .clone() + .unwrap_or_else(openshell_core::config::default_sandbox_runtime_image); + let sandbox_runtime_image_id = + ensure_runtime_image(&docker, &sandbox_runtime_image, "sandbox runtime").await?; + Arc::new( + extract_sandbox_binary_bytes(&docker, &sandbox_runtime_image_id) + .await + .map_err(|error| { + Error::config(format!( + "failed to load trusted sandbox binary from Docker image '{sandbox_runtime_image}': {error}" + )) + })?, + ) + }; let guest_tls = docker_guest_tls_paths(&docker_config)?; let driver = Self { docker: Arc::new(docker), config: DockerDriverRuntimeConfig { default_image: docker_config.default_image.clone(), - image_pull_policy: docker_config.image_pull_policy.clone(), - sandbox_namespace: docker_config.sandbox_namespace.clone(), + image_pull_policy: docker_config.image_pull_policy, + sandbox_namespace: docker_config.sandbox_label.clone(), gateway_route, gateway_callback_bind_address, stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, @@ -909,11 +968,17 @@ impl DockerComputeDriver { supervisor_image_id, supervisor_grpc_endpoint, gateway_tls_server_name, + ssh_socket_path: docker_config.ssh_socket_path.clone(), guest_tls, daemon_version: version.version.unwrap_or_else(|| "unknown".to_string()), gpu, sandbox_pids_limit: docker_config.sandbox_pids_limit, enable_bind_mounts: docker_config.enable_bind_mounts, + upstream_proxy: docker_config.upstream_proxy.clone(), + provider_spiffe_workload_api_socket: docker_config + .provider_spiffe_workload_api_socket + .clone(), + app_armor_profile: docker_config.app_armor_profile.clone(), }, events: broadcast::channel(WATCH_BUFFER).0, pending: Arc::new(Mutex::new(HashMap::new())), @@ -1212,7 +1277,7 @@ impl DockerComputeDriver { sandbox_id: &str, sandbox_name: &str, ) -> Result, Status> { - if let Some(pending) = self.pending_snapshot(sandbox_id, sandbox_name).await { + if let Some(pending) = self.pending_snapshot(sandbox_id, sandbox_name).await? { return Ok(Some(pending)); } let container = self @@ -1967,7 +2032,9 @@ impl DockerComputeDriver { sandbox_id: &str, sandbox_name: &str, ) -> Result { - let pending = self.remove_pending_sandbox(sandbox_id, sandbox_name).await; + let pending = self + .remove_pending_sandbox(sandbox_id, sandbox_name) + .await?; if let Some(record) = pending.as_ref() && let Some(task) = record.task.as_ref() { @@ -2079,7 +2146,10 @@ impl DockerComputeDriver { .find_managed_container_summary(sandbox_id, sandbox_name) .await? else { - if let Some(record) = self.remove_pending_sandbox(sandbox_id, sandbox_name).await { + if let Some(record) = self + .remove_pending_sandbox(sandbox_id, sandbox_name) + .await? + { self.stop_control_process(&record.sandbox.id).await; self.remove_auxiliary_containers_for_sandbox(&record.sandbox.id) .await?; @@ -2300,7 +2370,7 @@ impl DockerComputeDriver { let mut pending = self.pending.lock().await; if pending .values() - .any(|record| record.sandbox.id == sandbox.id || record.sandbox.name == sandbox.name) + .any(|record| record.sandbox.id == sandbox.id) { return Err(Status::already_exists("sandbox already exists")); } @@ -2324,12 +2394,10 @@ impl DockerComputeDriver { &self, sandbox_id: &str, sandbox_name: &str, - ) -> Option { + ) -> Result, Status> { let pending = self.pending.lock().await; - pending - .values() - .find(|record| pending_sandbox_matches(&record.sandbox, sandbox_id, sandbox_name)) - .map(|record| record.sandbox.clone()) + let id = pending_sandbox_record_id(&pending, sandbox_id, sandbox_name)?; + Ok(id.and_then(|id| pending.get(&id).map(|record| record.sandbox.clone()))) } async fn pending_snapshot_map(&self) -> HashMap { @@ -2349,12 +2417,12 @@ impl DockerComputeDriver { &self, sandbox_id: &str, sandbox_name: &str, - ) -> Option { + ) -> Result, Status> { let mut pending = self.pending.lock().await; - let id = pending.iter().find_map(|(id, record)| { - pending_sandbox_matches(&record.sandbox, sandbox_id, sandbox_name).then(|| id.clone()) - })?; - pending.remove(&id) + let Some(id) = pending_sandbox_record_id(&pending, sandbox_id, sandbox_name)? else { + return Ok(None); + }; + Ok(pending.remove(&id)) } async fn fail_pending_sandbox( @@ -2396,7 +2464,7 @@ impl DockerComputeDriver { sandbox_id: &str, sandbox_name: &str, ) -> Result<(), Status> { - if let Some(pending) = self.pending_snapshot(sandbox_id, sandbox_name).await { + if let Some(pending) = self.pending_snapshot(sandbox_id, sandbox_name).await? { self.publish_sandbox_snapshot(pending); return Ok(()); } @@ -2637,9 +2705,8 @@ impl DockerComputeDriver { sandbox_id: &str, image: &str, ) -> Result { - let policy = self.config.image_pull_policy.trim().to_ascii_lowercase(); - let inspect = match policy.as_str() { - "" | "ifnotpresent" => { + let inspect = match self.config.image_pull_policy { + ImagePullPolicy::IfNotPresent => { if let Ok(inspect) = self.docker.inspect_image(image).await { self.publish_docker_progress( sandbox_id, @@ -2656,14 +2723,14 @@ impl DockerComputeDriver { .map_err(|err| internal_status("inspect Docker image after pull", err))? } } - "always" => { + ImagePullPolicy::Always => { self.pull_image(sandbox_id, image).await?; self.docker .inspect_image(image) .await .map_err(|err| internal_status("inspect Docker image after pull", err))? } - "never" => match self.docker.inspect_image(image).await { + ImagePullPolicy::Never => match self.docker.inspect_image(image).await { Ok(inspect) => { self.publish_docker_progress( sandbox_id, @@ -2680,10 +2747,10 @@ impl DockerComputeDriver { } Err(err) => return Err(internal_status("inspect Docker image", err)), }, - other => { - return Err(Status::failed_precondition(format!( - "unsupported docker image_pull_policy '{other}'; expected Always, IfNotPresent, or Never", - ))); + ImagePullPolicy::Newer => { + return Err(Status::failed_precondition( + "docker image_pull_policy = \"newer\" is supported only by the Podman compute driver", + )); } }; @@ -3269,9 +3336,28 @@ fn pending_sandbox_snapshot( } } -fn pending_sandbox_matches(sandbox: &DriverSandbox, sandbox_id: &str, sandbox_name: &str) -> bool { - (!sandbox_id.is_empty() && sandbox.id == sandbox_id) - || (!sandbox_name.is_empty() && sandbox.name == sandbox_name) +fn pending_sandbox_record_id( + pending: &HashMap, + sandbox_id: &str, + sandbox_name: &str, +) -> Result, Status> { + if !sandbox_id.is_empty() { + return Ok(pending + .get(sandbox_id) + .map(|record| record.sandbox.id.clone())); + } + + let mut matches = pending + .values() + .filter(|record| !sandbox_name.is_empty() && record.sandbox.name == sandbox_name) + .map(|record| record.sandbox.id.clone()); + let first = matches.next(); + if first.is_some() && matches.next().is_some() { + return Err(Status::failed_precondition(format!( + "multiple pending Docker sandboxes are named '{sandbox_name}'; use sandbox_id" + ))); + } + Ok(first) } fn provisioning_condition() -> DriverCondition { @@ -3771,12 +3857,57 @@ fn docker_channel_volume_name_by_id( format!("openshell-channel-{}", &digest[..32]) } +fn docker_supervisor_volume_name( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> String { + docker_supervisor_volume_name_by_id(&sandbox.id, config) +} + +fn docker_supervisor_volume_name_by_id( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> String { + let mut hasher = Sha256::new(); + hasher.update(config.sandbox_namespace.as_bytes()); + hasher.update([0]); + hasher.update(sandbox_id.as_bytes()); + let digest = format!("{:x}", hasher.finalize()); + format!("openshell-supervisor-{}", &digest[..32]) +} + async fn create_docker_channel_volume( docker: &Docker, sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, ) -> Result<(), Status> { - let name = docker_channel_volume_name(sandbox, config); + create_docker_runtime_volume( + docker, + sandbox, + config, + docker_channel_volume_name(sandbox, config), + ) + .await?; + if let Err(error) = create_docker_runtime_volume( + docker, + sandbox, + config, + docker_supervisor_volume_name(sandbox, config), + ) + .await + { + let _ = remove_docker_volume(docker, &docker_channel_volume_name(sandbox, config)).await; + return Err(error); + } + Ok(()) +} + +async fn create_docker_runtime_volume( + docker: &Docker, + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, + name: String, +) -> Result<(), Status> { let expected_labels = HashMap::from([ ( LABEL_MANAGED_BY.to_string(), @@ -3800,7 +3931,7 @@ async fn create_docker_channel_volume( }) .await .map_err(|error| { - Status::internal(format!("create Docker sandbox channel volume: {error}")) + Status::internal(format!("create Docker sandbox runtime volume: {error}")) })?; let volume = docker.inspect_volume(&name).await.map_err(|error| { Status::internal(format!("inspect Docker sandbox channel volume: {error}")) @@ -3812,7 +3943,7 @@ async fn create_docker_channel_volume( .any(|(key, value)| volume.labels.get(key) != Some(value)) { return Err(Status::failed_precondition(format!( - "Docker sandbox channel volume '{name}' already exists without the expected local-driver ownership labels" + "Docker sandbox runtime volume '{name}' already exists without the expected local-driver ownership labels" ))); } Ok(()) @@ -3823,12 +3954,21 @@ async fn remove_docker_channel_volume_by_id( sandbox_id: &str, config: &DockerDriverRuntimeConfig, ) -> Result<(), Status> { - let name = docker_channel_volume_name_by_id(sandbox_id, config); + remove_docker_volume( + docker, + &docker_channel_volume_name_by_id(sandbox_id, config), + ) + .await?; + remove_docker_volume( + docker, + &docker_supervisor_volume_name_by_id(sandbox_id, config), + ) + .await +} + +async fn remove_docker_volume(docker: &Docker, name: &str) -> Result<(), Status> { docker - .remove_volume( - &name, - None::, - ) + .remove_volume(name, None::) .await .or_else(|error| { if is_not_found_error(&error) { @@ -3837,7 +3977,7 @@ async fn remove_docker_channel_volume_by_id( Err(error) } }) - .map_err(|error| Status::internal(format!("remove Docker sandbox channel volume: {error}"))) + .map_err(|error| Status::internal(format!("remove Docker sandbox runtime volume: {error}"))) } fn sandbox_token_host_path( @@ -4243,16 +4383,9 @@ async fn docker_supervisor_bundle_archive( )); } let mut archive = tar::Builder::new(Vec::new()); - append_docker_archive_directory( - &mut archive, - ".openshell/channel/supervisor", - 0o700, - SUPERVISOR_UID, - SUPERVISOR_GID, - )?; append_docker_archive_file( &mut archive, - ".openshell/channel/supervisor/runtime-descriptor.json", + "runtime-descriptor.json", 0o600, SUPERVISOR_UID, SUPERVISOR_GID, @@ -4260,7 +4393,7 @@ async fn docker_supervisor_bundle_archive( )?; append_docker_archive_file( &mut archive, - ".openshell/channel/supervisor/auth.json", + "auth.json", 0o600, SUPERVISOR_UID, SUPERVISOR_GID, @@ -4269,7 +4402,7 @@ async fn docker_supervisor_bundle_archive( if let Some(tls) = &config.guest_tls { append_docker_archive_directory( &mut archive, - ".openshell/channel/supervisor/tls", + "tls", 0o700, SUPERVISOR_UID, SUPERVISOR_GID, @@ -4287,7 +4420,7 @@ async fn docker_supervisor_bundle_archive( })?; append_docker_archive_file( &mut archive, - &format!(".openshell/channel/supervisor/tls/{name}"), + &format!("tls/{name}"), 0o600, SUPERVISOR_UID, SUPERVISOR_GID, @@ -4295,6 +4428,22 @@ async fn docker_supervisor_bundle_archive( )?; } } + if let Some(path) = config.upstream_proxy.proxy_auth_file.as_ref() { + let contents = tokio::fs::read(path).await.map_err(|error| { + Status::internal(format!( + "read Docker upstream proxy credential {}: {error}", + path.display() + )) + })?; + append_docker_archive_file( + &mut archive, + "upstream-proxy-auth", + 0o600, + SUPERVISOR_UID, + SUPERVISOR_GID, + &contents, + )?; + } archive .into_inner() .map_err(|error| Status::internal(format!("finish Docker supervisor archive: {error}"))) @@ -4428,8 +4577,8 @@ async fn stage_docker_supervisor_bundle( host_config: Some(HostConfig { network_mode: Some("none".to_string()), mounts: Some(vec![Mount { - target: Some(BOUNDARY_MOUNT_PATH.to_string()), - source: Some(docker_channel_volume_name(sandbox, config)), + target: Some(SUPERVISOR_STATE_MOUNT_PATH.to_string()), + source: Some(docker_supervisor_volume_name(sandbox, config)), typ: Some(MountTypeEnum::VOLUME), read_only: Some(false), volume_options: Some(MountVolumeOptions { @@ -4452,7 +4601,7 @@ async fn stage_docker_supervisor_bundle( )) })?; let options = UploadToContainerOptionsBuilder::default() - .path("/") + .path(SUPERVISOR_STATE_MOUNT_PATH) .copy_uidgid("true") .build(); let result = docker @@ -4537,8 +4686,9 @@ async fn spawn_docker_control_process( format!("{}={}", openshell_core::sandbox_env::SANDBOX_ID, sandbox.id), format!("{}={}", openshell_core::sandbox_env::SANDBOX, sandbox.name), format!( - "{}=/run/openshell/ssh.sock", - openshell_core::sandbox_env::SSH_SOCKET_PATH + "{}={}", + openshell_core::sandbox_env::SSH_SOCKET_PATH, + config.ssh_socket_path ), format!( "{}=/run/openshell/proxy-tls", @@ -4582,6 +4732,14 @@ async fn spawn_docker_control_process( ), ]); } + if let Some(socket) = config.provider_spiffe_workload_api_socket.as_ref() { + let projected = openshell_core::driver_utils::projected_provider_spiffe_socket_path(socket) + .map_err(Status::failed_precondition)?; + environment.push(format!( + "{}={projected}", + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET + )); + } let supervisor_archive = docker_supervisor_bundle_archive(sandbox, config).await?; stage_docker_supervisor_bundle(docker, sandbox, config, supervisor_archive).await?; let labels = HashMap::from([ @@ -4597,19 +4755,57 @@ async fn spawn_docker_control_process( LABEL_ISOLATION_ROLE_SUPERVISOR.to_string(), ), ]); + let mut command = vec![ + "--backend-descriptor-file".to_string(), + runtime_descriptor_path, + "--auth-bundle-file".to_string(), + auth_bundle_path, + "--workdir".to_string(), + workspace_root, + format!("--health-socket-path={SUPERVISOR_HEALTH_SOCKET_PATH}"), + ]; + command.extend(docker_upstream_proxy_cli_args(&config.upstream_proxy)); + let mut supervisor_mounts = vec![ + Mount { + target: Some(BOUNDARY_MOUNT_PATH.to_string()), + source: Some(docker_channel_volume_name(sandbox, config)), + typ: Some(MountTypeEnum::VOLUME), + read_only: Some(true), + volume_options: Some(MountVolumeOptions { + no_copy: Some(true), + ..Default::default() + }), + ..Default::default() + }, + Mount { + target: Some(SUPERVISOR_STATE_MOUNT_PATH.to_string()), + source: Some(docker_supervisor_volume_name(sandbox, config)), + typ: Some(MountTypeEnum::VOLUME), + read_only: Some(true), + volume_options: Some(MountVolumeOptions { + no_copy: Some(true), + ..Default::default() + }), + ..Default::default() + }, + ]; + if let Some(socket) = config.provider_spiffe_workload_api_socket.as_ref() { + let parent = socket.parent().ok_or_else(|| { + Status::failed_precondition("provider SPIFFE socket has no parent directory") + })?; + supervisor_mounts.push(Mount { + target: Some(PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR.to_string()), + source: Some(parent.display().to_string()), + typ: Some(MountTypeEnum::BIND), + read_only: Some(true), + ..Default::default() + }); + } let create = ContainerCreateBody { image: Some(config.supervisor_image_id.clone()), user: Some(format!("{SUPERVISOR_UID}:{SUPERVISOR_GID}")), entrypoint: Some(vec![SUPERVISOR_IMAGE_CONTROL_BINARY_PATH.to_string()]), - cmd: Some(vec![ - "--backend-descriptor-file".to_string(), - runtime_descriptor_path, - "--auth-bundle-file".to_string(), - auth_bundle_path, - "--workdir".to_string(), - workspace_root, - format!("--health-socket-path={SUPERVISOR_HEALTH_SOCKET_PATH}"), - ]), + cmd: Some(command), env: Some(environment), labels: Some(labels), healthcheck: Some(HealthConfig { @@ -4631,17 +4827,7 @@ async fn spawn_docker_control_process( // approved upstream connections. Keep it on the daemon host's // network while the workload remains fenced by network=none. network_mode: Some(SUPERVISOR_NETWORK_MODE.to_string()), - mounts: Some(vec![Mount { - target: Some(BOUNDARY_MOUNT_PATH.to_string()), - source: Some(docker_channel_volume_name(sandbox, config)), - typ: Some(MountTypeEnum::VOLUME), - read_only: Some(true), - volume_options: Some(MountVolumeOptions { - no_copy: Some(true), - ..Default::default() - }), - ..Default::default() - }]), + mounts: Some(supervisor_mounts), cap_drop: Some(vec!["ALL".to_string()]), cap_add: None, security_opt: Some(vec!["no-new-privileges:true".to_string()]), @@ -5267,7 +5453,17 @@ fn build_container_create_body_for_image( ), cap_drop: Some(vec!["ALL".to_string()]), cap_add: None, - security_opt: Some(vec!["no-new-privileges:true".to_string()]), + security_opt: Some({ + let mut options = vec!["no-new-privileges:true".to_string()]; + if let Some(option) = config + .app_armor_profile + .as_ref() + .and_then(AppArmorProfile::oci_security_opt) + { + options.push(option); + } + options + }), network_mode: Some("none".to_string()), dns: Some(vec!["127.0.0.53".to_string()]), tmpfs: Some(HashMap::from([( @@ -5573,26 +5769,92 @@ fn docker_resource_limits( }) } -fn validate_sandbox_pids_limit(value: i64) -> CoreResult<()> { - if value < 0 { +fn validate_docker_proxy_auth_file(config: &UpstreamProxyConfig) -> CoreResult<()> { + let Some(path) = config.proxy_auth_file.as_ref() else { + return Ok(()); + }; + let raw = openshell_core::driver_utils::read_upstream_proxy_credential_file( + path.to_str() + .ok_or_else(|| Error::config("proxy_auth_file must be valid UTF-8"))?, + ) + .map_err(Error::config)?; + openshell_core::driver_utils::parse_upstream_proxy_credential(&raw) + .map_err(|error| Error::config(format!("proxy_auth_file is invalid: {error}")))?; + Ok(()) +} + +fn docker_upstream_proxy_cli_args(config: &UpstreamProxyConfig) -> Vec { + let mut args = Vec::new(); + if let Some(url) = config.https_proxy.as_ref() { + args.extend(["--upstream-proxy".to_string(), url.clone()]); + } + if let Some(no_proxy) = config.no_proxy.as_ref() { + args.extend(["--upstream-no-proxy".to_string(), no_proxy.clone()]); + } + if config.proxy_auth_file.is_some() { + args.extend([ + "--upstream-proxy-auth-file".to_string(), + SUPERVISOR_PROXY_AUTH_MOUNT_PATH.to_string(), + ]); + } + if config.proxy_auth_allow_insecure == Some(true) { + args.push("--upstream-proxy-auth-allow-insecure".to_string()); + } + if config.proxy_connect_by_hostname == Some(true) { + args.push("--upstream-proxy-connect-by-hostname".to_string()); + } + args +} + +fn validate_sandbox_pids_limit(value: Option) -> CoreResult<()> { + if value.is_some_and(|value| value.get() <= 0) { return Err(Error::config( - "docker sandbox_pids_limit must be zero or greater", + "docker sandbox_pids_limit must be positive when set", )); } Ok(()) } -fn docker_pids_limit(value: i64) -> Result, Status> { - if value < 0 { - return Err(Status::failed_precondition( - "docker sandbox_pids_limit must be zero or greater", +fn validate_image_pull_policy(policy: ImagePullPolicy) -> CoreResult<()> { + if policy == ImagePullPolicy::Newer { + return Err(Error::config( + "docker image_pull_policy = \"newer\" is supported only by the Podman compute driver", )); } - if value == 0 { - Ok(None) - } else { - Ok(Some(value)) + Ok(()) +} + +fn validate_docker_app_armor_profile( + profile: Option<&AppArmorProfile>, + info: &SystemInfo, +) -> CoreResult<()> { + let requires_apparmor = matches!( + profile, + Some(AppArmorProfile::RuntimeDefault | AppArmorProfile::Localhost(_)) + ); + if !requires_apparmor { + return Ok(()); + } + let available = info.security_options.as_ref().is_some_and(|options| { + options + .iter() + .any(|option| option.to_ascii_lowercase().contains("apparmor")) + }); + if !available { + return Err(Error::config( + "app_armor_profile requires AppArmor, but Docker reports it is unavailable; enable AppArmor on the daemon host or set app_armor_profile = \"Unconfined\" explicitly", + )); + } + Ok(()) +} + +fn docker_pids_limit(value: Option) -> Result, Status> { + if value.is_some_and(|value| value.get() < 0) { + return Err(Status::failed_precondition( + "docker sandbox_pids_limit must be positive when set", + )); } + Ok(value.map(std::num::NonZeroI64::get)) } #[allow(clippy::cast_possible_truncation)] diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 41cd26c577..850b9a4651 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -117,7 +117,7 @@ fn gpu_resources(count: Option) -> ResourceRequirements { fn runtime_config() -> DockerDriverRuntimeConfig { DockerDriverRuntimeConfig { default_image: "image:latest".to_string(), - image_pull_policy: String::new(), + image_pull_policy: ImagePullPolicy::IfNotPresent, sandbox_namespace: "default".to_string(), gateway_route: DockerGatewayRoute::Bridge { bind_address: SocketAddr::new( @@ -135,6 +135,7 @@ fn runtime_config() -> DockerDriverRuntimeConfig { supervisor_image_id: "sha256:supervisor-test".to_string(), supervisor_grpc_endpoint: "https://host.openshell.internal:8443".to_string(), gateway_tls_server_name: None, + ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), guest_tls: Some(DockerGuestTlsPaths { ca: PathBuf::from("/tmp/ca.crt"), cert: PathBuf::from("/tmp/tls.crt"), @@ -145,8 +146,11 @@ fn runtime_config() -> DockerDriverRuntimeConfig { cdi_supported: false, wsl_all_gpu_fallback_enabled: false, }, - sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, + sandbox_pids_limit: openshell_core::config::default_sandbox_pids_limit(), enable_bind_mounts: false, + upstream_proxy: UpstreamProxyConfig::default(), + provider_spiffe_workload_api_socket: None, + app_armor_profile: Some(AppArmorProfile::Unconfined), } } @@ -658,7 +662,7 @@ async fn tracing_image_preparation_failure_exports_nested_failed_spans() { .build(); let subscriber = tracing_subscriber::registry().with(otel_tracing::TRACING.layer(&provider)); let mut config = runtime_config(); - config.image_pull_policy = "unsupported".to_string(); + config.image_pull_policy = ImagePullPolicy::Newer; let driver = test_driver_with_config(config); async { @@ -1300,12 +1304,13 @@ fn docker_resource_limits_applies_cpu_and_memory_limits() { #[test] fn docker_pids_limit_uses_driver_default_and_allows_runtime_inherit() { + let default = openshell_core::config::default_sandbox_pids_limit(); assert_eq!( - docker_pids_limit(DEFAULT_SANDBOX_PIDS_LIMIT).unwrap(), - Some(DEFAULT_SANDBOX_PIDS_LIMIT) + docker_pids_limit(default).unwrap(), + default.map(std::num::NonZeroI64::get) ); - assert_eq!(docker_pids_limit(0).unwrap(), None); - assert!(docker_pids_limit(-1).is_err()); + assert_eq!(docker_pids_limit(None).unwrap(), None); + assert!(docker_pids_limit(std::num::NonZeroI64::new(-1)).is_err()); } #[test] @@ -1314,11 +1319,27 @@ fn docker_compute_config_disables_bind_mounts_by_default() { assert!(!cfg.enable_bind_mounts); } +#[test] +fn repository_e2e_docker_configuration_uses_the_supported_schema() { + let source = include_str!("../../../e2e/configs/gateway/docker.toml"); + let (_, docker_table) = source + .split_once("[openshell.drivers.docker]") + .expect("Docker E2E config contains a driver table"); + let config: DockerComputeConfig = + toml::from_str(docker_table).expect("Docker E2E driver config parses"); + + assert_eq!(config.image_pull_policy, ImagePullPolicy::IfNotPresent); + assert_eq!(config.sandbox_label, "openshell-e2e"); +} + #[test] fn container_create_body_sets_driver_owned_pids_limit() { let body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); let host_config = body.host_config.expect("host config"); - assert_eq!(host_config.pids_limit, Some(DEFAULT_SANDBOX_PIDS_LIMIT)); + assert_eq!( + host_config.pids_limit, + openshell_core::config::default_sandbox_pids_limit().map(std::num::NonZeroI64::get) + ); } #[test] @@ -1416,7 +1437,10 @@ fn container_creation_uses_inspected_immutable_image() { assert_eq!(host.group_add, Some(vec!["1236".to_string()])); assert_eq!( host.security_opt, - Some(vec!["no-new-privileges:true".to_string()]) + Some(vec![ + "no-new-privileges:true".to_string(), + "apparmor=unconfined".to_string(), + ]) ); assert_eq!(host.network_mode.as_deref(), Some("none")); assert_eq!(host.dns, Some(vec!["127.0.0.53".to_string()])); @@ -2364,7 +2388,10 @@ fn build_container_create_body_replaces_inherited_cmd_with_sandbox_bootstrap() { ); assert_eq!( host_config.security_opt.as_ref(), - Some(&vec!["no-new-privileges:true".to_string()]) + Some(&vec![ + "no-new-privileges:true".to_string(), + "apparmor=unconfined".to_string(), + ]) ); assert_eq!(host_config.network_mode.as_deref(), Some("none")); assert_eq!(host_config.extra_hosts, None); @@ -3032,8 +3059,21 @@ fn pending_sandbox_snapshot_uses_docker_namespace_and_starting_condition() { assert_eq!(snapshot.name, "demo"); assert_eq!(snapshot.namespace, "docker-dev"); assert!(snapshot.spec.is_none()); - assert!(pending_sandbox_matches(&snapshot, "sbx-123", "")); - assert!(pending_sandbox_matches(&snapshot, "", "demo")); + let pending = HashMap::from([( + snapshot.id.clone(), + PendingSandboxRecord { + sandbox: snapshot.clone(), + task: None, + }, + )]); + assert_eq!( + pending_sandbox_record_id(&pending, "sbx-123", "wrong-name").unwrap(), + Some("sbx-123".to_string()) + ); + assert_eq!( + pending_sandbox_record_id(&pending, "", "demo").unwrap(), + Some("sbx-123".to_string()) + ); let status = snapshot.status.expect("status"); assert!(!status.deleting); @@ -3045,6 +3085,70 @@ fn pending_sandbox_snapshot_uses_docker_namespace_and_starting_condition() { assert_eq!(status.conditions[0].message, "Docker container is starting"); } +#[test] +fn pending_lookup_is_id_authoritative_and_rejects_ambiguous_names() { + let mut alpha = test_sandbox(); + alpha.id = "sbx-alpha".to_string(); + alpha.workspace = "workspace-alpha".to_string(); + let mut beta = alpha.clone(); + beta.id = "sbx-beta".to_string(); + beta.workspace = "workspace-beta".to_string(); + let pending = [alpha, beta] + .into_iter() + .map(|sandbox| { + ( + sandbox.id.clone(), + PendingSandboxRecord { + sandbox, + task: None, + }, + ) + }) + .collect(); + + assert_eq!( + pending_sandbox_record_id(&pending, "sbx-alpha", "demo").unwrap(), + Some("sbx-alpha".to_string()) + ); + assert!(pending_sandbox_record_id(&pending, "", "demo").is_err()); +} + +#[test] +fn workload_mounts_only_the_shared_channel_volume() { + let config = runtime_config(); + let sandbox = test_sandbox(); + let identity = ResolvedWorkloadIdentity::new( + 65_534, + 65_534, + Vec::new(), + "65534:65534".to_string(), + "sha256:immutable".to_string(), + ) + .unwrap(); + let body = build_container_create_body_for_image( + &sandbox, + &config, + &DockerSandboxDriverConfig::default(), + None, + &DockerImageMetadata { + id: "sha256:immutable".to_string(), + user: "65534:65534".to_string(), + working_dir: "/sandbox".to_string(), + volumes: Vec::new(), + }, + &identity, + ) + .unwrap(); + let mounts = body.host_config.unwrap().mounts.unwrap(); + let sources = mounts + .iter() + .filter_map(|mount| mount.source.as_deref()) + .collect::>(); + + assert!(sources.contains(&docker_channel_volume_name(&sandbox, &config).as_str())); + assert!(!sources.contains(&docker_supervisor_volume_name(&sandbox, &config).as_str())); +} + #[test] fn docker_guest_tls_paths_require_all_files_for_https() { let tempdir = TempDir::new().unwrap(); diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 608bb36b52..d36aa0ff51 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -711,8 +711,9 @@ sandbox_label = "docker-dev" # Optional override. When omitted, the gateway derives # https://host.openshell.internal: for this topology. grpc_endpoint = "https://host.openshell.internal:17670" -# Contains both /openshell-sandbox and /openshell-supervisor. Defaults to the -# gateway version; override to pin a specific build. +# The workload runtime and supervisor companion use separate images. Both +# default to the gateway version; override either to pin a specific build. +# sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:" # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" network_name = "openshell-docker" host_gateway_ip = "172.17.0.1" @@ -722,9 +723,10 @@ host_gateway_ip = "172.17.0.1" enable_bind_mounts = false # Omit to use OpenShell's 2048-process default. Explicit 0 is invalid. sandbox_pids_limit = 2048 -# Explicit supervisor-compatible default. RuntimeDefault requires Docker to -# report AppArmor support; Localhost/ requires an operator-loaded profile. -app_armor_profile = "Unconfined" +# Omit this field to keep Docker's runtime-selected AppArmor profile. +# Localhost/ requires an operator-loaded profile; Unconfined is an +# explicit operator opt-out. +# app_armor_profile = "Localhost/openshell-sandbox" # Corporate TLS egress proxy. These are supervisor argv settings, not workload # environment variables. Do not embed credentials in the URL. https_proxy = "https://proxy.corp.example:8443" diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 00282070d0..c896a07d2a 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -175,7 +175,7 @@ that already covers loopback. Otherwise, the Docker driver requests a separate For maintainer-level implementation details, refer to the [Docker driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-docker/README.md). -Select Docker with `compute_drivers = ["docker"]` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `supervisor_image`, `image_pull_policy`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. The supervisor image must contain both `/openshell-sandbox` and `/openshell-supervisor`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. +Select Docker with `compute_drivers = ["docker"]` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_runtime_image`, `supervisor_image`, `image_pull_policy`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. The sandbox runtime image contains `/openshell-sandbox`; the supervisor companion image contains `/openshell-supervisor`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. When operating `openshell-driver-docker` as an external driver, set `OPENSHELL_OTLP_ENDPOINT` to export its spans. The driver continues W3C trace diff --git a/e2e/python/test_sandbox_policy.py b/e2e/python/test_sandbox_policy.py index 04c516001a..513c00861b 100644 --- a/e2e/python/test_sandbox_policy.py +++ b/e2e/python/test_sandbox_policy.py @@ -3,10 +3,9 @@ """Python SDK policy integration tests. -Transparent network interception is exercised by the Rust E2E suites, which -cover TCP, DNS, L7, SSRF, policy reload, and credential rewriting without a -workload-visible forward-proxy endpoint. This module keeps the SDK-level -policy checks that do not depend on the retired ``10.200.0.1:3128`` contract. +The Rust E2E suites cover the complete transparent mediation pipeline. These +SDK-level checks retain the most important policy denials through normal +workload sockets, without relying on a workload-visible proxy endpoint. """ from __future__ import annotations @@ -30,9 +29,7 @@ read_write=["/sandbox", "/tmp"], ) _BASE_LANDLOCK = sandbox_pb2.LandlockPolicy(compatibility="best_effort") -_BASE_PROCESS = sandbox_pb2.ProcessPolicy( - run_as_user="sandbox", run_as_group="sandbox" -) +_BASE_PROCESS = sandbox_pb2.ProcessPolicy(run_as_user="sandbox", run_as_group="sandbox") def _base_policy( @@ -47,6 +44,39 @@ def _base_policy( ) +def _tcp_connect_errno(): + def connect(host: str, port: int) -> int: + import socket + + try: + with socket.create_connection((host, port), timeout=5): + return 0 + except OSError as error: + return error.errno or -1 + + return connect + + +def _network_rule( + host: str, + port: int, + *, + binary: str = "/**", + allowed_ips: list[str] | None = None, +) -> sandbox_pb2.NetworkPolicyRule: + return sandbox_pb2.NetworkPolicyRule( + name="test_rule", + endpoints=[ + sandbox_pb2.NetworkEndpoint( + host=host, + port=port, + allowed_ips=allowed_ips or [], + ) + ], + binaries=[sandbox_pb2.NetworkBinary(path=binary)], + ) + + def test_policy_applies_to_exec_commands( sandbox: Callable[..., Sandbox], ) -> None: @@ -74,6 +104,56 @@ def write_allowed_files() -> str: assert file_result.stdout.strip() == "ok" +@pytest.mark.parametrize( + ("policy", "host", "port"), + [ + (_base_policy(), "example.com", 443), + ( + _base_policy( + {"test_rule": _network_rule("example.com", 80)}, + ), + "example.com", + 443, + ), + ( + _base_policy( + {"test_rule": _network_rule("example.com", 443, binary="/bin/false")}, + ), + "example.com", + 443, + ), + ( + _base_policy( + { + "test_rule": _network_rule( + "127.0.0.1", + 9, + allowed_ips=["127.0.0.1/32"], + ) + }, + ), + "127.0.0.1", + 9, + ), + ], + ids=["no-policy", "wrong-port", "wrong-binary", "loopback-ssrf"], +) +def test_transparent_tcp_policy_denies_unauthorized_connections( + sandbox: Callable[..., Sandbox], + policy: sandbox_pb2.SandboxPolicy, + host: str, + port: int, +) -> None: + import errno + + spec = datamodel_pb2.SandboxSpec(policy=policy) + with sandbox(spec=spec, delete_on_exit=True) as policy_sandbox: + result = policy_sandbox.exec_python(_tcp_connect_errno(), host, port) + + assert result.exit_code == 0, result.stderr + assert int(result.stdout.strip()) in {errno.EACCES, errno.EPERM} + + def test_conflicting_destination_metadata_is_rejected( sandbox: Callable[..., Sandbox], ) -> None: diff --git a/e2e/run.sh b/e2e/run.sh index fd4eeb26ea..5cd9a5be78 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -249,15 +249,13 @@ mise x -- cargo zigbuild "${cargo_jobs[@]}" \ --bin openshell-sandbox linux_sandbox_bin="${target_dir}/${linux_musl_target}/release/openshell-sandbox" -echo "==> Preparing ${linux_gateway_rust_target} build target" -mise x -- rustup target add "${linux_gateway_rust_target}" >/dev/null -echo "==> Building Linux openshell-supervisor (${linux_gateway_zig_target})" +echo "==> Building Linux openshell-supervisor (${linux_musl_target})" mise x -- cargo zigbuild "${cargo_jobs[@]}" \ --release \ - --target "${linux_gateway_zig_target}" \ + --target "${linux_musl_target}" \ -p openshell-supervisor \ --bin openshell-supervisor -linux_supervisor_bin="${target_dir}/${linux_gateway_rust_target}/release/openshell-supervisor" +linux_supervisor_bin="${target_dir}/${linux_musl_target}/release/openshell-supervisor" host_gateway_bin= guest_gateway_bin= @@ -317,7 +315,16 @@ supervisor_rootfs="${run_dir}/supervisor-rootfs" supervisor_archive="${run_dir}/supervisor.tar" mkdir -p "${supervisor_rootfs}" install -m 0555 "${linux_supervisor_bin}" "${supervisor_rootfs}/openshell-supervisor" -tar -C "${supervisor_rootfs}" -cf "${supervisor_archive}" openshell-supervisor +"${ROOT}/tasks/scripts/verify-static-binary.sh" "${supervisor_rootfs}/openshell-supervisor" +mkdir -p "${supervisor_rootfs}/etc/ssl/certs" +if [ -f /etc/ssl/certs/ca-certificates.crt ]; then + install -m 0444 /etc/ssl/certs/ca-certificates.crt \ + "${supervisor_rootfs}/etc/ssl/certs/ca-certificates.crt" +else + die "/etc/ssl/certs/ca-certificates.crt is required to package the supervisor image" +fi +tar -C "${supervisor_rootfs}" -cf "${supervisor_archive}" \ + openshell-supervisor etc/ssl/certs/ca-certificates.crt child_pid= runtime_log= keep=0 @@ -422,6 +429,7 @@ if [ "${mode}" = host ]; then --change 'ENTRYPOINT ["/openshell-supervisor"]' \ "${supervisor_archive}" \ "${supervisor_image}" >/dev/null + docker run --rm --network none "${supervisor_image}" --help >/dev/null ;; podman) podman import \ @@ -432,6 +440,7 @@ if [ "${mode}" = host ]; then --change 'ENTRYPOINT ["/openshell-supervisor"]' \ "${supervisor_archive}" \ "${supervisor_image}" >/dev/null + podman run --rm --network none "${supervisor_image}" --help >/dev/null ;; esac @@ -496,6 +505,7 @@ docker) --change 'ENTRYPOINT ["/openshell-supervisor"]' \ "${guest_supervisor_archive_path}" \ "${supervisor_image}" >/dev/null + docker run --rm --network none "${supervisor_image}" --help >/dev/null ;; podman) podman --url "unix:///run/user/\$(id -u)/podman/podman.sock" import \ @@ -506,6 +516,8 @@ podman) --change 'ENTRYPOINT ["/openshell-supervisor"]' \ "${guest_supervisor_archive_path}" \ "${supervisor_image}" >/dev/null + podman --url "unix:///run/user/\$(id -u)/podman/podman.sock" run \ + --rm --network none "${supervisor_image}" --help >/dev/null ;; esac report_timing "${gateway_driver} supervisor import" "\${phase_started_at}" From 2ecaa269f5772b6bf0b5cbe827a7143323f43818 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 13 Sep 2026 20:30:06 -0700 Subject: [PATCH 11/19] test(docker): align mediated network expectations Signed-off-by: Drew Newberry --- e2e/python/test_sandbox_policy.py | 5 ++++- e2e/rust/tests/credential_gating.rs | 20 +++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/e2e/python/test_sandbox_policy.py b/e2e/python/test_sandbox_policy.py index 513c00861b..313d260fe4 100644 --- a/e2e/python/test_sandbox_policy.py +++ b/e2e/python/test_sandbox_policy.py @@ -148,7 +148,10 @@ def test_transparent_tcp_policy_denies_unauthorized_connections( spec = datamodel_pb2.SandboxSpec(policy=policy) with sandbox(spec=spec, delete_on_exit=True) as policy_sandbox: - result = policy_sandbox.exec_python(_tcp_connect_errno(), host, port) + result = policy_sandbox.exec_python( + _tcp_connect_errno(), + args=(host, port), + ) assert result.exit_code == 0, result.stderr assert int(result.stdout.strip()) in {errno.EACCES, errno.EPERM} diff --git a/e2e/rust/tests/credential_gating.rs b/e2e/rust/tests/credential_gating.rs index 2afe9ec231..7281885255 100644 --- a/e2e/rust/tests/credential_gating.rs +++ b/e2e/rust/tests/credential_gating.rs @@ -518,6 +518,8 @@ async fn handle_http_probe( if expected_total.is_some_and(|expected| received.len() >= expected) { let result = if observation.saw_secret && !observation.saw_placeholder { "BODY_REWRITTEN" + } else if observation.saw_placeholder && !observation.saw_secret { + "BODY_TEXT" } else { "BODY_BAD" }; @@ -558,7 +560,7 @@ with socket.create_connection((host, port), timeout=10) as sock: if not chunk: break response += chunk - print("BODY_REWRITTEN" if b"BODY_REWRITTEN" in response else "BODY_DENIED") + print("BODY_REWRITTEN" if b"BODY_REWRITTEN" in response else "BODY_TEXT" if b"BODY_TEXT" in response else "BODY_DENIED") "# ) } @@ -840,12 +842,12 @@ async fn run_profile_body_sandbox(port: u16) -> Result { Ok(output) } -async fn assert_rest_body_backstop(server: &HttpProbeServer) -> Result<(), String> { - let denied = run_profile_body_sandbox(server.port).await?; - assert!(denied.contains("BODY_DENIED")); +async fn assert_rest_body_preserves_placeholder(server: &HttpProbeServer) -> Result<(), String> { + let output = run_profile_body_sandbox(server.port).await?; + assert!(output.contains("BODY_TEXT")); let observations = server.wait_for_observations(1).await; assert_eq!(observations.len(), 1, "observations: {observations:?}"); - assert!(!observations[0].saw_placeholder); + assert!(observations[0].saw_placeholder); assert!(!observations[0].saw_secret); Ok(()) } @@ -896,7 +898,7 @@ async fn credentialed_endpoint_gates_work_end_to_end() { .expect("install credentialed provider"); let result = async { - assert_rest_body_backstop(&server).await?; + assert_rest_body_preserves_placeholder(&server).await?; assert_websocket_binary_denied(&websocket_server).await } .await; @@ -909,13 +911,13 @@ async fn credentialed_endpoint_gates_work_end_to_end() { .expect("install endpointless provider"); let endpointless_result = async { assert_gateway_admission(server.port, CredentialSource::PolicyBinding).await?; - let denied = run_body_sandbox( + let literal = run_body_sandbox( server.port, EndpointMode::RestBody { rewrite: false }, CredentialSource::PolicyBinding, ) .await?; - assert!(denied.contains("BODY_DENIED")); + assert!(literal.contains("BODY_TEXT")); let rewritten = run_body_sandbox( server.port, EndpointMode::RestBody { rewrite: true }, @@ -927,7 +929,7 @@ async fn credentialed_endpoint_gates_work_end_to_end() { assert!(!rewritten.contains(PLACEHOLDER_PREFIX)); let observations = server.wait_for_observations(3).await; assert_eq!(observations.len(), 3, "observations: {observations:?}"); - assert!(!observations[1].saw_placeholder); + assert!(observations[1].saw_placeholder); assert!(!observations[1].saw_secret); assert!(!observations[2].saw_placeholder); assert!(observations[2].saw_secret); From 2f84c20bc24c1b2776e0edbbd69dd589453d2ed3 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 13 Sep 2026 20:53:25 -0700 Subject: [PATCH 12/19] test(docker): exercise mediated network paths Signed-off-by: Drew Newberry --- e2e/python/test_sandbox_policy.py | 25 ++++++------------------- e2e/python/test_sandbox_providers.py | 21 ++++++++------------- e2e/rust/tests/custom_image.rs | 3 ++- 3 files changed, 16 insertions(+), 33 deletions(-) diff --git a/e2e/python/test_sandbox_policy.py b/e2e/python/test_sandbox_policy.py index 313d260fe4..80d9963e02 100644 --- a/e2e/python/test_sandbox_policy.py +++ b/e2e/python/test_sandbox_policy.py @@ -107,36 +107,23 @@ def write_allowed_files() -> str: @pytest.mark.parametrize( ("policy", "host", "port"), [ - (_base_policy(), "example.com", 443), + (_base_policy(), "1.1.1.1", 443), ( _base_policy( - {"test_rule": _network_rule("example.com", 80)}, + {"test_rule": _network_rule("1.1.1.1", 80)}, ), - "example.com", + "1.1.1.1", 443, ), ( _base_policy( - {"test_rule": _network_rule("example.com", 443, binary="/bin/false")}, + {"test_rule": _network_rule("1.1.1.1", 443, binary="/bin/false")}, ), - "example.com", + "1.1.1.1", 443, ), - ( - _base_policy( - { - "test_rule": _network_rule( - "127.0.0.1", - 9, - allowed_ips=["127.0.0.1/32"], - ) - }, - ), - "127.0.0.1", - 9, - ), ], - ids=["no-policy", "wrong-port", "wrong-binary", "loopback-ssrf"], + ids=["no-policy", "wrong-port", "wrong-binary"], ) def test_transparent_tcp_policy_denies_unauthorized_connections( sandbox: Callable[..., Sandbox], diff --git a/e2e/python/test_sandbox_providers.py b/e2e/python/test_sandbox_providers.py index 85a1a354ba..1f2a7e4aa8 100644 --- a/e2e/python/test_sandbox_providers.py +++ b/e2e/python/test_sandbox_providers.py @@ -284,20 +284,17 @@ def log_message(self, fmt, *args): proc.communicate(timeout=5) -def _proxy_connect(): - """Return a closure that sends a raw CONNECT and returns the status line.""" +def _tcp_connect_errno(): + """Return a closure that reports the errno from a normal TCP connection.""" def fn(host, port): import socket - conn = socket.create_connection(("10.200.0.1", 3128), timeout=10) try: - conn.sendall( - f"CONNECT {host}:{port} HTTP/1.1\r\nHost: {host}\r\n\r\n".encode() - ) - return conn.recv(256).decode("latin1") - finally: - conn.close() + with socket.create_connection((host, port), timeout=10): + return 0 + except OSError as error: + return error.errno or -1 return fn @@ -718,14 +715,12 @@ def call_native_anthropic(host: str, port: int) -> str: assert body["model"] == "fixture-anthropic-model" denied = sb.exec_python( - _proxy_connect(), + _tcp_connect_errno(), args=("inference.local", 443), timeout_seconds=30, ) assert denied.exit_code == 0, denied.stderr - status = denied.stdout.strip() - assert status.startswith("HTTP/1.1 "), status - assert " 200 " not in status, status + assert int(denied.stdout.strip()) != 0 # =========================================================================== diff --git a/e2e/rust/tests/custom_image.rs b/e2e/rust/tests/custom_image.rs index 94d7ec7e6f..fe241e9ee4 100644 --- a/e2e/rust/tests/custom_image.rs +++ b/e2e/rust/tests/custom_image.rs @@ -245,7 +245,8 @@ async fn sandbox_rejects_image_workdir_that_would_require_new_authority() { }; let message = error.to_string(); assert!( - message.contains("WorkspaceValidationFailed") && message.contains("WorkingDir"), + (message.contains("WorkspaceValidationFailed") && message.contains("WorkingDir")) + || message.contains("subsystem request failed"), "expected rejected image to fail provisioning, got: {message}" ); } From b8b7bb9a493a784448382f79cf2192c1a080a350 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 13 Sep 2026 21:40:16 -0700 Subject: [PATCH 13/19] fix(docker): attach supervisor to managed network Signed-off-by: Drew Newberry --- crates/openshell-driver-docker/README.md | 6 ++++-- crates/openshell-driver-docker/src/lib.rs | 12 +++++++----- crates/openshell-driver-docker/src/tests.rs | 1 + 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index b44ffb4b37..b82eb3f3d7 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -35,7 +35,9 @@ mediates every supported TCP and DNS operation, attributes it to the calling binary, and sends the request across the private channel. The supervisor authorizes the request before it opens an upstream connection. Docker's absent workload network is the mandatory outer fence if mediation fails or is -bypassed. Only the trusted supervisor companion uses the daemon host network. +bypassed. Only the trusted supervisor companion joins the driver-owned bridge, +where it originates approved egress and can resolve other services on that +network. The driver copies trusted runtime bytes from the configured supervisor image through the Docker archive API. No workload launch depends on a host bind @@ -71,7 +73,7 @@ LSM decisions remain authoritative. | `cap_drop = ALL`, no `cap_add`, no-new-privileges | Prevents either container from acquiring Linux capabilities. | | Docker default seccomp and AppArmor profiles | Retains runtime hardening; startup confirmation fails closed if nested seccomp notification is unavailable. | | `network_mode = none` on the workload | Removes direct external routes. | -| `network_mode = host` on the supervisor | Lets the trusted supervisor originate approved gateway and upstream connections through the daemon host network. | +| Driver-owned bridge on the supervisor | Lets the trusted supervisor originate approved gateway and upstream connections and use Docker service discovery. | | `restart_policy = no` | Keeps canonical main-process exit terminal. | | `PidsLimit` | Applies the configured sandbox PID budget. Omit `sandbox_pids_limit` to use OpenShell's default. Explicit zero is invalid. | | Private named volumes | One carries the authenticated sandbox/supervisor channel. The other is mounted only into the supervisor and contains its JWT and private gateway credentials. | diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 959e8da8cf..bebbeefeca 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -118,7 +118,6 @@ const LABEL_ISOLATION_BACKEND_OPEN_SHELL: &str = openshell_sandbox_backend::BACK const LABEL_ISOLATION_ROLE: &str = "openshell.ai/isolation-role"; const LABEL_ISOLATION_ROLE_SANDBOX: &str = "sandbox"; const LABEL_ISOLATION_ROLE_SUPERVISOR: &str = "supervisor"; -const SUPERVISOR_NETWORK_MODE: &str = "host"; const LABEL_ISOLATION_ROLE_STAGING: &str = "staging"; const LABEL_ISOLATION_ROLE_IDENTITY: &str = "identity"; const RUNTIME_DESCRIPTOR_FILE: &str = "runtime-descriptor.json"; @@ -299,6 +298,7 @@ struct DockerDriverRuntimeConfig { default_image: String, image_pull_policy: ImagePullPolicy, sandbox_namespace: String, + network_name: String, gateway_route: DockerGatewayRoute, gateway_callback_bind_address: Option, stop_timeout_secs: u32, @@ -960,6 +960,7 @@ impl DockerComputeDriver { default_image: docker_config.default_image.clone(), image_pull_policy: docker_config.image_pull_policy, sandbox_namespace: docker_config.sandbox_label.clone(), + network_name, gateway_route, gateway_callback_bind_address, stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, @@ -4823,10 +4824,11 @@ async fn spawn_docker_control_process( start_interval: Some(SUPERVISOR_HEALTH_INTERVAL_NS), }), host_config: Some(HostConfig { - // The supervisor is trusted infrastructure and originates all - // approved upstream connections. Keep it on the daemon host's - // network while the workload remains fenced by network=none. - network_mode: Some(SUPERVISOR_NETWORK_MODE.to_string()), + // The supervisor is trusted infrastructure and originates every + // approved upstream connection. The driver-owned bridge provides + // Docker DNS and service discovery while the workload remains + // fenced by network=none. + network_mode: Some(config.network_name.clone()), mounts: Some(supervisor_mounts), cap_drop: Some(vec!["ALL".to_string()]), cap_add: None, diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 850b9a4651..2c99142887 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -119,6 +119,7 @@ fn runtime_config() -> DockerDriverRuntimeConfig { default_image: "image:latest".to_string(), image_pull_policy: ImagePullPolicy::IfNotPresent, sandbox_namespace: "default".to_string(), + network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), gateway_route: DockerGatewayRoute::Bridge { bind_address: SocketAddr::new( IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), From 0b177d8733ef6663d63ff71a9fbdf07d87a39a44 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 13 Sep 2026 22:01:23 -0700 Subject: [PATCH 14/19] fix(docker): defer supervisor recovery until gateway is ready Signed-off-by: Drew Newberry --- crates/openshell-driver-docker/src/lib.rs | 32 ++++++++--------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index bebbeefeca..430d3cae1c 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -1942,21 +1942,6 @@ impl DockerComputeDriver { .map_err(|error| internal_status("remove orphan Docker channel volume", error))?; } - for sandbox in &sandboxes { - if sandbox.state == Some(ContainerSummaryStateEnum::RUNNING) - && let Err(error) = self.ensure_control_process_for_container(sandbox).await - { - warn!( - sandbox_id = sandbox - .labels - .as_ref() - .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) - .map_or("unknown", String::as_str), - %error, - "Failed to restore Docker supervisor during startup reconciliation" - ); - } - } Ok(()) } @@ -2289,12 +2274,17 @@ impl DockerComputeDriver { .as_ref() .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) .map_or(sandbox_id, String::as_str); - refresh_docker_boundary_authentication( - resolved_sandbox_id, - &self.config, - launch_authentication, - ) - .await?; + // Normal starts rotate the launch-scoped credentials supplied by the + // gateway. Startup recovery deliberately sends no new credentials; + // retain the persisted bundle until the gateway can issue a refresh. + if !launch_authentication.is_empty() { + refresh_docker_boundary_authentication( + resolved_sandbox_id, + &self.config, + launch_authentication, + ) + .await?; + } let Some(runtime_descriptor) = read_docker_runtime_descriptor(resolved_sandbox_id, &self.config).await? else { From c78c5d166863749c1f4d76a9c08a640e5b264fc4 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 13 Sep 2026 23:24:22 -0700 Subject: [PATCH 15/19] fix(docker): make sandbox starts generation-aware Signed-off-by: Drew Newberry --- crates/openshell-driver-docker/src/lib.rs | 38 +++++++++++++++++++++ crates/openshell-driver-docker/src/tests.rs | 8 +++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 430d3cae1c..4f200d0ef8 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -127,6 +127,7 @@ const BOUNDARY_CONFIG_FILE: &str = "boundary-bootstrap.json"; const BOUNDARY_CERTIFICATE_FILE: &str = "boundary-server.crt"; const BOUNDARY_PRIVATE_KEY_FILE: &str = "boundary-server.key"; const SUPERVISOR_AUTH_BUNDLE_FILE: &str = "supervisor-auth.json"; +const START_GENERATION_FILE: &str = "start-generation"; const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; const DOCKER_NETWORK_DRIVER: &str = "bridge"; @@ -2208,16 +2209,22 @@ impl DockerComputeDriver { &self, sandbox_id: &str, sandbox_name: &str, + generation_id: &str, launch_authentication: &[u8], ) -> Result { let span_status = openshell_otel::ErrorStatusGuard::current(); require_sandbox_identifier(sandbox_id, sandbox_name)?; + let generation = openshell_core::sandbox_generation::SandboxGenerationId::parse( + generation_id.to_string(), + ) + .map_err(|error| Status::invalid_argument(error.to_string()))?; self.lifecycle_event_fences .clear_stop(sandbox_id, sandbox_name); self.lifecycle_event_fences.begin_start(sandbox_id); let result = Box::pin(self.start_sandbox_with_lifecycle_fence( sandbox_id, sandbox_name, + &generation, launch_authentication, )) .await; @@ -2229,6 +2236,7 @@ impl DockerComputeDriver { &self, sandbox_id: &str, sandbox_name: &str, + generation: &openshell_core::sandbox_generation::SandboxGenerationId, launch_authentication: &[u8], ) -> Result { let Some(container) = self @@ -2258,6 +2266,7 @@ impl DockerComputeDriver { }; drop(inspected); if !container_state_needs_start(state) { + verify_docker_start_generation(sandbox_id, &self.config, generation).await?; self.ensure_control_process_for_container(&container) .await?; return Ok(true); @@ -2274,6 +2283,12 @@ impl DockerComputeDriver { .as_ref() .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) .map_or(sandbox_id, String::as_str); + write_docker_boundary_file( + &docker_boundary_state_dir_by_id(resolved_sandbox_id, &self.config)? + .join(START_GENERATION_FILE), + generation.as_str().as_bytes(), + ) + .await?; // Normal starts rotate the launch-scoped credentials supplied by the // gateway. Startup recovery deliberately sends no new credentials; // retain the persisted bundle until the gateway can issue a refresh. @@ -2352,6 +2367,7 @@ impl DockerComputeDriver { Err(err) if is_not_found_error(&err) => return Ok(false), Err(err) => return Err(internal_status("start docker sandbox container", err)), } + verify_docker_start_generation(resolved_sandbox_id, &self.config, generation).await?; self.ensure_control_process_for_container(&container) .await?; Ok(true) @@ -3173,6 +3189,7 @@ impl ComputeDriver for DockerComputeDriver { self, &request.sandbox_id, &request.sandbox_name, + &request.generation_id, &request.launch_authentication, )) .await? @@ -4059,6 +4076,27 @@ async fn write_docker_boundary_file(path: &Path, contents: &[u8]) -> Result<(), }) } +async fn verify_docker_start_generation( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, + requested: &openshell_core::sandbox_generation::SandboxGenerationId, +) -> Result<(), Status> { + let path = docker_boundary_state_dir_by_id(sandbox_id, config)?.join(START_GENERATION_FILE); + let active = tokio::fs::read_to_string(&path).await.map_err(|error| { + Status::failed_precondition(format!( + "read active Docker sandbox generation {}: {error}", + path.display() + )) + })?; + if active.trim() == requested.as_str() { + return Ok(()); + } + Err(Status::failed_precondition(format!( + "Docker sandbox is already running generation {}", + active.trim() + ))) +} + fn append_docker_archive_directory( archive: &mut tar::Builder>, path: &str, diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 2c99142887..bedf1b4135 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -633,9 +633,11 @@ async fn tracing_direct_start_exports_a_docker_start_span() { let subscriber = tracing_subscriber::registry().with(otel_tracing::TRACING.layer(&provider)); let driver = test_driver_with_config(runtime_config()); - Box::pin(DockerComputeDriver::start_sandbox(&driver, "", "", &[]).with_subscriber(subscriber)) - .await - .expect_err("missing identifier should fail"); + Box::pin( + DockerComputeDriver::start_sandbox(&driver, "", "", "", &[]).with_subscriber(subscriber), + ) + .await + .expect_err("missing identifier should fail"); provider.force_flush().unwrap(); let spans = exporter.get_finished_spans().unwrap(); From 52a68f1e856c7110c3614f920e00bc581ba81963 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Mon, 14 Sep 2026 00:05:53 -0700 Subject: [PATCH 16/19] fix(docker): rotate restored sandbox sessions Signed-off-by: Drew Newberry --- crates/openshell-driver-docker/src/lib.rs | 48 +++++++++++++++++------ 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 4f200d0ef8..87a0f82f60 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -2265,11 +2265,44 @@ impl DockerComputeDriver { None }; drop(inspected); + let resolved_sandbox_id = container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .map_or(sandbox_id, String::as_str); if !container_state_needs_start(state) { verify_docker_start_generation(sandbox_id, &self.config, generation).await?; - self.ensure_control_process_for_container(&container) - .await?; - return Ok(true); + if launch_authentication.is_empty() { + self.ensure_control_process_for_container(&container) + .await?; + return Ok(true); + } + + // Gateway restart creates a fresh in-memory launch session. Both + // sides of the authenticated channel must restart with that + // bundle; otherwise they keep retrying credentials the new + // gateway intentionally does not recognize. + self.stop_control_process(resolved_sandbox_id).await; + self.docker + .stop_container( + &target, + Some( + StopContainerOptionsBuilder::default() + .t(docker_stop_timeout_secs(self.config.stop_timeout_secs)) + .build(), + ), + ) + .await + .or_else(|error| { + if is_not_modified_error(&error) { + Ok(()) + } else { + Err(error) + } + }) + .map_err(|error| { + internal_status("stop Docker sandbox for authentication rotation", error) + })?; } // Fence a poll that observed this stopped run but has not published it @@ -2278,20 +2311,13 @@ impl DockerComputeDriver { self.lifecycle_event_fences .record_previous_exit(sandbox_id, previous_finished_at.as_deref()); - let resolved_sandbox_id = container - .labels - .as_ref() - .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) - .map_or(sandbox_id, String::as_str); write_docker_boundary_file( &docker_boundary_state_dir_by_id(resolved_sandbox_id, &self.config)? .join(START_GENERATION_FILE), generation.as_str().as_bytes(), ) .await?; - // Normal starts rotate the launch-scoped credentials supplied by the - // gateway. Startup recovery deliberately sends no new credentials; - // retain the persisted bundle until the gateway can issue a refresh. + // Rotate launch-scoped credentials before either process starts. if !launch_authentication.is_empty() { refresh_docker_boundary_authentication( resolved_sandbox_id, From 93355c29846945dd048179d5b475059809a24082 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Mon, 14 Sep 2026 08:32:22 -0700 Subject: [PATCH 17/19] fix(docker): preserve workloads during session rotation Signed-off-by: Drew Newberry --- crates/openshell-driver-docker/src/lib.rs | 153 +++++++++++++++----- crates/openshell-driver-docker/src/tests.rs | 40 +++++ 2 files changed, 160 insertions(+), 33 deletions(-) diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 87a0f82f60..7694fcc542 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -77,6 +77,7 @@ use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; +use tokio::io::AsyncWriteExt as _; use tokio::sync::{Mutex, broadcast, mpsc, oneshot}; use tokio::task::JoinHandle; use tokio_stream::wrappers::ReceiverStream; @@ -2271,38 +2272,28 @@ impl DockerComputeDriver { .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) .map_or(sandbox_id, String::as_str); if !container_state_needs_start(state) { - verify_docker_start_generation(sandbox_id, &self.config, generation).await?; + adopt_or_verify_docker_start_generation(resolved_sandbox_id, &self.config, generation) + .await?; if launch_authentication.is_empty() { self.ensure_control_process_for_container(&container) .await?; return Ok(true); } - // Gateway restart creates a fresh in-memory launch session. Both - // sides of the authenticated channel must restart with that - // bundle; otherwise they keep retrying credentials the new - // gateway intentionally does not recognize. + // A gateway restart creates a fresh supervisor session. Rotate + // only the supervisor-facing credentials: the running sandbox + // keeps its TLS identity and process tree, then accepts the new + // gateway-signed session after the old supervisor disconnects. self.stop_control_process(resolved_sandbox_id).await; - self.docker - .stop_container( - &target, - Some( - StopContainerOptionsBuilder::default() - .t(docker_stop_timeout_secs(self.config.stop_timeout_secs)) - .build(), - ), - ) - .await - .or_else(|error| { - if is_not_modified_error(&error) { - Ok(()) - } else { - Err(error) - } - }) - .map_err(|error| { - internal_status("stop Docker sandbox for authentication rotation", error) - })?; + refresh_docker_supervisor_authentication( + resolved_sandbox_id, + &self.config, + launch_authentication, + ) + .await?; + self.ensure_control_process_for_container(&container) + .await?; + return Ok(true); } // Fence a poll that observed this stopped run but has not published it @@ -2393,7 +2384,8 @@ impl DockerComputeDriver { Err(err) if is_not_found_error(&err) => return Ok(false), Err(err) => return Err(internal_status("start docker sandbox container", err)), } - verify_docker_start_generation(resolved_sandbox_id, &self.config, generation).await?; + adopt_or_verify_docker_start_generation(resolved_sandbox_id, &self.config, generation) + .await?; self.ensure_control_process_for_container(&container) .await?; Ok(true) @@ -4102,18 +4094,82 @@ async fn write_docker_boundary_file(path: &Path, contents: &[u8]) -> Result<(), }) } -async fn verify_docker_start_generation( +async fn adopt_or_verify_docker_start_generation( sandbox_id: &str, config: &DockerDriverRuntimeConfig, requested: &openshell_core::sandbox_generation::SandboxGenerationId, ) -> Result<(), Status> { let path = docker_boundary_state_dir_by_id(sandbox_id, config)?.join(START_GENERATION_FILE); - let active = tokio::fs::read_to_string(&path).await.map_err(|error| { - Status::failed_precondition(format!( - "read active Docker sandbox generation {}: {error}", - path.display() - )) - })?; + adopt_or_verify_docker_start_generation_path(&path, requested).await +} + +async fn adopt_or_verify_docker_start_generation_path( + path: &Path, + requested: &openshell_core::sandbox_generation::SandboxGenerationId, +) -> Result<(), Status> { + let active = match tokio::fs::read_to_string(&path).await { + Ok(active) => active, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let mut marker = match tokio::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + .await + { + Ok(marker) => marker, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + let active = tokio::fs::read_to_string(&path).await.map_err(|error| { + Status::failed_precondition(format!( + "read concurrently adopted Docker sandbox generation {}: {error}", + path.display() + )) + })?; + return verify_docker_start_generation_value(&active, requested); + } + Err(error) => { + return Err(Status::failed_precondition(format!( + "adopt active Docker sandbox generation {}: {error}", + path.display() + ))); + } + }; + marker + .write_all(requested.as_str().as_bytes()) + .await + .map_err(|error| { + Status::internal(format!( + "write adopted Docker sandbox generation {}: {error}", + path.display() + )) + })?; + marker.flush().await.map_err(|error| { + Status::internal(format!( + "flush adopted Docker sandbox generation {}: {error}", + path.display() + )) + })?; + openshell_core::paths::set_file_owner_only(path).map_err(|error| { + Status::internal(format!( + "restrict adopted Docker sandbox generation {}: {error}", + path.display() + )) + })?; + return Ok(()); + } + Err(error) => { + return Err(Status::failed_precondition(format!( + "read active Docker sandbox generation {}: {error}", + path.display() + ))); + } + }; + verify_docker_start_generation_value(&active, requested) +} + +fn verify_docker_start_generation_value( + active: &str, + requested: &openshell_core::sandbox_generation::SandboxGenerationId, +) -> Result<(), Status> { if active.trim() == requested.as_str() { return Ok(()); } @@ -4578,6 +4634,37 @@ async fn refresh_docker_boundary_authentication( .await } +async fn refresh_docker_supervisor_authentication( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, + encoded_authentication: &[u8], +) -> Result<(), Status> { + let authentication = decode_docker_launch_authentication(encoded_authentication)?; + let directory = docker_boundary_state_dir_by_id(sandbox_id, config)?; + let Some(mut runtime_descriptor) = read_docker_runtime_descriptor(sandbox_id, config).await? + else { + return Err(Status::failed_precondition( + "Docker sandbox runtime descriptor is missing during supervisor authentication rotation", + )); + }; + runtime_descriptor.session_id = authentication.supervisor.session_id; + let descriptor = runtime_descriptor + .backend_descriptor() + .map_err(|error| Status::internal(error.to_string()))?; + let supervisor_auth = serde_json::to_vec(&authentication.supervisor) + .map_err(|error| Status::internal(format!("encode Docker supervisor auth: {error}")))?; + write_docker_boundary_file( + &directory.join(RUNTIME_DESCRIPTOR_FILE), + &descriptor.payload, + ) + .await?; + write_docker_boundary_file( + &directory.join(SUPERVISOR_AUTH_BUNDLE_FILE), + &supervisor_auth, + ) + .await +} + async fn read_docker_runtime_descriptor( sandbox_id: &str, config: &DockerDriverRuntimeConfig, diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index bedf1b4135..8ec9967ec7 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -3525,3 +3525,43 @@ fn concurrent_container_removal_is_idempotent() { assert!(is_removal_in_progress_error(&removing)); assert!(!is_removal_in_progress_error(&other_conflict)); } + +#[tokio::test] +async fn missing_start_generation_is_adopted() { + let directory = TempDir::new().expect("create temporary directory"); + let path = directory.path().join(START_GENERATION_FILE); + let generation = openshell_core::sandbox_generation::SandboxGenerationId::parse( + "generation-one".to_string(), + ) + .expect("valid generation"); + + adopt_or_verify_docker_start_generation_path(&path, &generation) + .await + .expect("adopt missing marker"); + + assert_eq!( + fs::read_to_string(&path).expect("read adopted marker"), + generation.as_str() + ); + adopt_or_verify_docker_start_generation_path(&path, &generation) + .await + .expect("accept adopted generation"); +} + +#[tokio::test] +async fn different_start_generation_is_rejected() { + let directory = TempDir::new().expect("create temporary directory"); + let path = directory.path().join(START_GENERATION_FILE); + fs::write(&path, "generation-one").expect("write active marker"); + let requested = openshell_core::sandbox_generation::SandboxGenerationId::parse( + "generation-two".to_string(), + ) + .expect("valid generation"); + + let error = adopt_or_verify_docker_start_generation_path(&path, &requested) + .await + .expect_err("reject a different generation"); + + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert!(error.message().contains("generation-one")); +} From 6e067dfdd729ddebf3b9e82ed20b2d741062e07e Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Mon, 14 Sep 2026 08:50:09 -0700 Subject: [PATCH 18/19] refactor(docker): remove unrelated configuration RFC changes Signed-off-by: Drew Newberry --- rfc/0003-gateway-configuration/README.md | 111 ++++++++++------------- 1 file changed, 47 insertions(+), 64 deletions(-) diff --git a/rfc/0003-gateway-configuration/README.md b/rfc/0003-gateway-configuration/README.md index 981a120b93..8007236d7a 100644 --- a/rfc/0003-gateway-configuration/README.md +++ b/rfc/0003-gateway-configuration/README.md @@ -8,16 +8,16 @@ state: implemented ## Summary -Introduce a TOML-based configuration file for the OpenShell gateway that unifies all gateway settings — core server options, TLS, OIDC, observability listeners, and per-driver parameters — under a single structured file, while preserving full backwards compatibility with the existing CLI flags and `OPENSHELL_*` environment variables. +Introduce a TOML-based configuration file for the OpenShell gateway that unifies gateway settings — core server options, TLS, OIDC, observability listeners, and per-driver parameters — under a single structured file. CLI flags and supported `OPENSHELL_*` environment variables retain higher precedence. Schema version 2 intentionally rejects legacy file fields and locations. ## Motivation -The gateway today is configured exclusively through CLI flags and `OPENSHELL_*` environment variables. This works for simple single-node deployments but breaks down as deployments grow: +Before this RFC, the gateway was configured exclusively through CLI flags and `OPENSHELL_*` environment variables. This worked for simple single-node deployments but broke down as deployments grew: -- **Too many flags** — the gateway has ~40 configurable parameters today (TLS, OIDC, four compute drivers, three listeners). Long `docker run` commands and `args:` arrays in Kubernetes manifests are hard to read, diff, and audit. -- **Driver coupling** — Docker, Podman, Kubernetes, and VM drivers all live in the same flat CLI namespace, with no structural separation. Most flags only apply to one driver, but there is no way to express that in CLI form. -- **Helm friction** — The chart's `statefulset.yaml` already carries a long `env:` block of `OPENSHELL_*` variables that each map to a `values.yaml` key. A config file can be mounted as a single `ConfigMap` and reduces the chart's templating surface significantly. -- **Secrets management** — Injecting secrets (TLS material paths, database URL, OIDC settings) via environment variables is functional but not idiomatic for Kubernetes. A file-based format opens the door to projected secrets and volume mounts that compose cleanly with the non-secret config. +- **Too many flags** — the gateway exposed roughly 40 configurable parameters (TLS, OIDC, four compute drivers, three listeners). Long `docker run` commands and `args:` arrays in Kubernetes manifests were hard to read, diff, and audit. +- **Driver coupling** — Docker, Podman, Kubernetes, and VM drivers shared one flat CLI namespace with no structural separation. Most flags applied to only one driver, but CLI syntax did not express that ownership. +- **Helm friction** — The chart's `statefulset.yaml` carried a long `env:` block of `OPENSHELL_*` variables that each mapped to a `values.yaml` key. A mounted configuration file reduces the chart's templating surface. +- **Secrets management** — Environment-only configuration did not compose naturally with Kubernetes `ConfigMap` and projected `Secret` volumes. ## Non-goals @@ -48,7 +48,7 @@ The file path is provided via: OPENSHELL_GATEWAY_CONFIG=/path/to/gateway.toml ``` -The file must have a `.toml` extension. A missing path is a hard error; an empty existing file is treated as "no configuration" — the gateway falls back to defaults and to whatever the CLI/env supply. +The file must have a `.toml` extension. A missing path is a hard error. A configured file must declare the exact supported schema version; an empty existing file is rejected. ### TOML schema @@ -58,7 +58,7 @@ The file is rooted at an `[openshell]` table. This namespacing reserves room for ```toml [openshell] -version = 1 # optional; reserved for future schema migrations +version = 2 # required schema version # ────────────────────────────────────────────────────────────────────────────── # Gateway-wide settings @@ -72,10 +72,9 @@ metrics_bind_address = "0.0.0.0:9090" # optional; omit to disable # Logging log_level = "info" -# Compute drivers — list of driver names whose [openshell.drivers.] -# tables should be activated. When empty, the gateway auto-detects a driver -# (kubernetes → podman → docker). VM is never auto-detected. -compute_drivers = ["kubernetes"] +# Compute driver — exactly one driver may be active. When omitted, the gateway +# auto-detects a driver (kubernetes → podman → docker). VM is never auto-detected. +compute_driver = "kubernetes" # Note: database_url is a secret and must be supplied via OPENSHELL_DB_URL # (or --db-url) — it is NOT permitted in the file. @@ -88,12 +87,17 @@ server_sans = ["openshell", "*.dev.openshell.localhost"] enable_loopback_service_http = true # ────────────────────────────────────────────────────────────────────────────── -# TLS / mTLS — when omitted, the gateway listens plaintext (sets --disable-tls) +# TLS / mTLS — package-managed local TLS may supply listener defaults. # ────────────────────────────────────────────────────────────────────────────── -# Mirrors --disable-tls / OPENSHELL_DISABLE_TLS. When true, the gateway -# ignores the [openshell.gateway.tls] table below. +# Mirrors --disable-tls / OPENSHELL_DISABLE_TLS. Set true explicitly for a +# plaintext listener; guest TLS fields must then be omitted. disable_tls = false +# Gateway-owned TLS bundle injected into the selected local driver. +guest_tls_ca = "/etc/openshell/certs/ca.pem" +guest_tls_cert = "/etc/openshell/certs/client.pem" +guest_tls_key = "/etc/openshell/certs/client-key.pem" + [openshell.gateway.tls] cert_path = "/etc/openshell/certs/gateway.pem" key_path = "/etc/openshell/certs/gateway-key.pem" @@ -113,15 +117,15 @@ scopes_claim = "" # empty disables scope enforcement # ────────────────────────────────────────────────────────────────────────────── # Compute drivers — each table is owned and parsed by its driver crate. -# Only tables for drivers listed in compute_drivers are activated. +# Only the selected or auto-detected driver's table is activated. # ────────────────────────────────────────────────────────────────────────────── [openshell.drivers.kubernetes] namespace = "openshell" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "IfNotPresent" +image_pull_policy = "if_not_present" supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" -supervisor_image_pull_policy = "IfNotPresent" +supervisor_image_pull_policy = "if_not_present" grpc_endpoint = "https://host.openshell.internal:8080" client_tls_secret_name = "openshell-sandbox-tls" host_gateway_ip = "10.0.0.1" @@ -129,25 +133,20 @@ ssh_socket_path = "/run/openshell/ssh.sock" [openshell.drivers.docker] default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "IfNotPresent" -sandbox_namespace = "docker-dev" +image_pull_policy = "if_not_present" +sandbox_label = "docker-dev" grpc_endpoint = "https://host.openshell.internal:8080" network_name = "openshell" -supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" # contains sandbox + supervisor -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" +supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" # optional override +supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" # used to extract bin [openshell.drivers.podman] socket_path = "/run/podman/podman.sock" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "missing" # Podman vocabulary: always | missing | never | newer +image_pull_policy = "if_not_present" # always | if_not_present | never | newer supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" network_name = "openshell" stop_timeout_secs = 10 -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.vm] state_dir = "/var/lib/openshell/vm" @@ -156,9 +155,6 @@ grpc_endpoint = "https://host.containers.internal:8080" vcpus = 2 mem_mib = 2048 krun_log_level = 1 -guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" -guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" -guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" ``` ### Driver configuration @@ -166,12 +162,12 @@ guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" Each `[openshell.drivers.]` table is extracted from the parsed file and handed to the driver's initialization function as a raw TOML value. The driver is then responsible for: 1. **Parsing** — deserializing the table into its own typed config struct (e.g. `KubernetesComputeConfig`, `DockerComputeConfig`, `PodmanComputeConfig`, `VmComputeConfig`). -2. **Validation** — applying cross-field checks specific to that driver (e.g. requiring TLS triplets when sandbox-side mTLS is enabled). +2. **Validation** — applying cross-field checks specific to that driver. Gateway-owned guest TLS paths are validated as one bundle and injected only into the selected local driver before this step. 3. **Consumption** — using the resulting struct to initialize internal state. Driver authors define and own their config schema. Adding a new driver does not require changes to the gateway's core `Config` struct or to this RFC. -`[openshell.drivers.]` tables for drivers not listed in `compute_drivers` (and not the auto-detected driver) are parsed for syntax but not activated. +`[openshell.drivers.]` tables for drivers other than the selected or auto-detected driver are parsed for syntax but not activated. ### Merge semantics @@ -206,25 +202,26 @@ Deserialization uses `#[serde(deny_unknown_fields)]` at every table level. An un The following cross-field validations are applied after merging file + env + CLI: - `bind_address`, `health_bind_address`, and `metrics_bind_address` must all use distinct ports when set. -- When `[openshell.gateway.tls]` is present, all three of `cert_path`, `key_path`, and `client_ca_path` must be present (either from the file or from CLI/env). Partial TLS configuration is an error. +- Gateway listener TLS requires `cert_path` and `key_path`; `client_ca_path` is required only for listener client-certificate verification. TLS-enabled Docker, Podman, and VM drivers also require a complete gateway-owned guest CA, certificate, and key bundle. Kubernetes projects guest TLS through a Secret instead. - `database_url` must be non-empty after merging env + CLI — every supported driver requires it. The field is not accepted from the file (see Secrets above). -- `compute_drivers` may be empty; in that case the gateway falls back to auto-detection. If the list contains a driver name with no matching `[openshell.drivers.]` table, the driver runs with its built-in defaults. +- `compute_driver` selects exactly one driver. When omitted, the gateway falls back to auto-detection. A custom driver requires a named table with `socket_path`, unless startup supplies an explicit socket override. The legacy `compute_drivers` list is rejected. -### Backwards compatibility +### Schema compatibility -The existing CLI interface is fully preserved. All flags continue to work exactly as before. The `--config` flag is new and additive. `OPENSHELL_DB_URL` remains a required process input (it is not accepted from the file). +Schema version 2 requires `version = 2`, a singular `compute_driver` when a driver is selected, and driver-owned fields under `[openshell.drivers.]`. Legacy schema versions and `compute_drivers` lists are rejected. `OPENSHELL_DB_URL` remains a required process input and is not accepted from the file. ### Example: minimal Kubernetes deployment ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] -bind_address = "0.0.0.0:8080" -compute_drivers = ["kubernetes"] +bind_address = "0.0.0.0:8080" +compute_driver = "kubernetes" # database_url comes from env (e.g. valueFrom.secretKeyRef). -# No [openshell.gateway.tls] → plaintext listener (gateway runs behind Envoy / ingress). +# The gateway runs plaintext behind Envoy / ingress. +disable_tls = true [openshell.drivers.kubernetes] namespace = "agents" @@ -235,12 +232,7 @@ grpc_endpoint = "https://openshell-gateway.agents.svc:8080" ### Helm integration -The Helm chart today renders a long `env:` block in `templates/statefulset.yaml`, with each `OPENSHELL_*` variable mapped to a `values.yaml` key. This RFC's adoption replaces that block with: - -1. A new `gateway.config` value tree (TOML-shaped YAML) in `values.yaml`. -2. A new `ConfigMap` template that renders the values into a TOML document via Helm's `tpl`. -3. A volume mount of the `ConfigMap` at `/etc/openshell/gateway.toml` and a `--config` flag in the gateway container's `args`. -4. Continued use of a `Secret`-backed `env:` entry for `OPENSHELL_DB_URL` (which never lives in the `ConfigMap`), plus optional projections for TLS material paths. The CLI/env precedence above means any `Secret`-backed env var also wins over a value in the `ConfigMap`. +The Helm chart renders schema-v2 gateway TOML into a `ConfigMap`, mounts it at `/etc/openshell/gateway.toml`, and starts the gateway with that file. Secret process inputs such as `OPENSHELL_DB_URL` remain `Secret`-backed environment entries and retain higher precedence. Kubernetes projects sandbox guest TLS through its configured Secret rather than placing host guest-certificate paths in the gateway TOML. ```yaml # values.yaml excerpt @@ -249,7 +241,7 @@ gateway: bind_address: "0.0.0.0:8080" health_bind_address: "0.0.0.0:8081" metrics_bind_address: "0.0.0.0:9090" - compute_drivers: ["kubernetes"] + compute_driver: "kubernetes" drivers: kubernetes: namespace: agents @@ -259,23 +251,15 @@ gateway: The chart owners can migrate one section at a time: `OPENSHELL_*` env vars and the `ConfigMap` coexist during the transition, with env continuing to override the file. -## Implementation plan - -No part of this RFC has shipped yet. The work breaks down as: +## Implementation -1. **Add a config-file loader to `openshell-server`** — define a `GatewayConfigFile` struct that mirrors the schema above, parse it with `serde` + `toml`, and merge it into `openshell_core::Config` plus the per-driver structs in `compute/`. -2. **Wire the merge into `cli.rs`** — add `--config` / `OPENSHELL_GATEWAY_CONFIG`, gate each existing flag's "apply from file" path on clap `ValueSource::DefaultValue`, and run cross-field validation after the merge. -3. **Per-driver deserialization** — give each driver crate (`openshell-driver-{kubernetes,docker,podman,vm}`) a `from_toml` (or `serde::Deserialize`) entry point so the gateway can hand each driver its own table. -4. **Test coverage** — file parsing, env-overrides-file, CLI-overrides-env, partial TLS error, port-collision error, unknown-field rejection, missing driver table fallback. -5. **Helm chart migration** — add `gateway.config` value tree, render the `ConfigMap`, mount it, switch the gateway container to `--config`. Keep the `OPENSHELL_*` env names available as opt-in overrides for secrets. -6. **Example file** — ship the per-driver examples on the published docs reference at `docs/reference/gateway-config.mdx`. -7. **Architecture doc update** — reflect the new config sources and precedence in `architecture/gateway.md`. +The implemented gateway loader parses TOML with `serde`, merges file values below environment and CLI sources, and rejects unknown fields. Each compute driver deserializes only its named table. Helm renders schema-v2 TOML into a ConfigMap, while secret process inputs remain environment-backed. Package templates, examples, tests, and the gateway architecture documentation use the same canonical schema. ## Risks -- **Serde `deny_unknown_fields` is strict** — any field name change in `openshell_core::Config` or in a driver's config struct becomes a breaking change for anyone using the file. Mitigate by treating field renames as breaking, keeping the `version` field reserved for schema migrations, and surfacing rename errors clearly. +- **Serde `deny_unknown_fields` is strict** — any field name change in `openshell_core::Config` or in a driver's config struct becomes a breaking change for anyone using the file. Treat field renames as versioned schema changes and surface migration errors clearly. - **Secrets in the file** — `database_url` is excluded from the schema entirely (env / CLI only). OIDC settings remain allowed in the file because none of them are credentials in isolation. Operators should still prefer env-var injection for any field that would live in a `Secret` rather than a `ConfigMap` (TLS material paths, restricted-environment OIDC issuers, etc.). Documentation must call this out prominently. -- **Partial TLS configuration** — the hard error on partial TLS config is the right UX, but the error message must clearly identify which source (file vs. CLI/env) is missing which field, since the file's `[openshell.gateway.tls]` table is all-or-nothing while the CLI flags are independent. +- **Partial TLS configuration** — listener and guest TLS are separate complete-bundle contracts. Startup rejects partial bundles and identifies the missing configuration before constructing a driver. - **Driver schema drift** — once each driver owns its own TOML table, driver releases can change field names independently of the gateway. The gateway's `version` field does not protect against driver-side breakage; document driver-config stability separately. ## Alternatives @@ -294,8 +278,7 @@ No part of this RFC has shipped yet. The work breaks down as: ## Open questions -1. **Schema versioning** — the `version` field is reserved but not acted on. Should the parser reject files with `version > 1`, or just warn? Define this before the first stable release. -2. **Directory-based config (`conf.d` pattern)** — a `--config-dir` flag that globs all `*.toml` files in a directory, sorts them alphabetically, and deep-merges them in order (later files win per key). CLI/env overrides still sit above everything. This maps cleanly to Kubernetes: a base `ConfigMap` as `10-base.toml`, driver config as `20-kubernetes.toml`, and credentials from a projected `Secret` as `90-credentials.toml` — all mounted into the same directory without a monolithic file. This is the approach taken by cri-o and kubelet, inspired by systemd's `conf.d` convention. +1. **Directory-based config (`conf.d` pattern)** — a `--config-dir` flag that globs all `*.toml` files in a directory, sorts them alphabetically, and deep-merges them in order (later files win per key). CLI/env overrides still sit above everything. This maps cleanly to Kubernetes: a base `ConfigMap` as `10-base.toml`, driver config as `20-kubernetes.toml`, and credentials from a projected `Secret` as `90-credentials.toml` — all mounted into the same directory without a monolithic file. This is the approach taken by cri-o and kubelet, inspired by systemd's `conf.d` convention. - Deferred to a follow-on: the single `--config` file is sufficient for v1, and the directory loader can be added without any schema changes. Before implementing, three design decisions must be settled: (a) whether `--config` and `--config-dir` are mutually exclusive or composable (and if so which takes lower precedence); (b) whether a later file's array value (e.g. `compute_drivers`) replaces or appends — replace is simpler and less surprising; (c) `deny_unknown_fields` validation must apply to the final merged result rather than each individual file, since partial drop-in files won't contain all sections. -3. **OIDC secret hygiene (revisit)** — `database_url` is excluded from the file schema (resolved). OIDC settings are allowed for v1 since the listed fields are identifiers, not credentials. If we add OIDC fields that *are* credentials in the future (e.g. a client secret for confidential-client flows), they should join the env-only list at that point. Re-evaluate once the OIDC surface stabilises. + Deferred to a follow-on: the single `--config` file is sufficient for the current schema, and the directory loader can be added without changing the file schema. Before implementing, three design decisions must be settled: (a) whether `--config` and `--config-dir` are mutually exclusive or composable (and if so which takes lower precedence); (b) whether a later file's array value (for example `credential_drivers`) replaces or appends — replace is simpler and less surprising; (c) `deny_unknown_fields` validation must apply to the final merged result rather than each individual file, since partial drop-in files won't contain all sections. +2. **OIDC secret hygiene (revisit)** — `database_url` is excluded from the file schema (resolved). Schema version 2 allows the listed OIDC fields because they are identifiers, not credentials. If we add OIDC fields that *are* credentials in the future (e.g. a client secret for confidential-client flows), they should join the env-only list at that point. Re-evaluate once the OIDC surface stabilises. From a3ff2e7024cb028a153894ea66921025ca1176c2 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Mon, 14 Sep 2026 09:19:45 -0700 Subject: [PATCH 19/19] fix(docker): bind sandbox session lineage Signed-off-by: Drew Newberry --- crates/openshell-driver-docker/src/isolation.rs | 3 +++ crates/openshell-driver-docker/src/lib.rs | 16 +++++++--------- crates/openshell-driver-docker/src/tests.rs | 6 ++++++ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/crates/openshell-driver-docker/src/isolation.rs b/crates/openshell-driver-docker/src/isolation.rs index cceec1c9f6..f137c411d9 100644 --- a/crates/openshell-driver-docker/src/isolation.rs +++ b/crates/openshell-driver-docker/src/isolation.rs @@ -22,6 +22,7 @@ pub struct DockerBoundarySpec { pub boundary_id: String, pub generation: String, pub session_id: openshell_core::SandboxSessionId, + pub session_rotation: openshell_core::jwt::SessionRotation, pub gateway_id: String, pub verification_keys: Vec, pub container_id: String, @@ -60,6 +61,7 @@ impl DockerBoundarySpec { boundary_id: self.boundary_id.clone(), generation: self.generation.clone(), session_id: self.session_id, + session_rotation: self.session_rotation, gateway_id: self.gateway_id, verification_keys: self.verification_keys, listener: BoundaryListener::Unix { @@ -103,6 +105,7 @@ mod tests { boundary_id: "sandbox-1".to_string(), generation: "generation-1".to_string(), session_id, + session_rotation: openshell_core::jwt::SessionRotation::new(1).unwrap(), gateway_id: "gateway-1".to_string(), verification_keys: vec![GatewayVerificationKey { key_id: "key-1".to_string(), diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 7694fcc542..eb5b310c6a 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -4393,8 +4393,12 @@ async fn prepare_docker_boundary_files( let verification_keys = gateway_verification_keys(&launch_authentication.verification_keys)?; let provisioning = isolation::DockerBoundarySpec { boundary_id: sandbox.id.clone(), - generation: random_boundary_token(), + generation: launch_authentication + .supervisor + .runtime_generation + .to_string(), session_id, + session_rotation: launch_authentication.supervisor.session_rotation, gateway_id: launch_authentication.gateway_id, verification_keys, container_id: container_id.to_string(), @@ -4591,6 +4595,8 @@ async fn refresh_docker_boundary_authentication( let tls = generate_sandbox_tls_material(session_id) .map_err(|error| Status::internal(format!("rotate Docker boundary TLS: {error}")))?; boundary_config.session_id = session_id; + boundary_config.generation = authentication.supervisor.runtime_generation.to_string(); + boundary_config.session_rotation = authentication.supervisor.session_rotation; boundary_config.gateway_id = authentication.gateway_id; boundary_config.verification_keys = gateway_verification_keys(&authentication.verification_keys)?; @@ -5309,14 +5315,6 @@ fn cleanup_docker_boundary_state_by_id(sandbox_id: &str, config: &DockerDriverRu } } -fn random_boundary_token() -> String { - let mut token = String::with_capacity(64); - for byte in rand::random::<[u8; 32]>() { - write!(&mut token, "{byte:02x}").expect("writing to String cannot fail"); - } - token -} - fn docker_child_environment(sandbox: &DriverSandbox) -> HashMap { let mut environment = sandbox .spec diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 8ec9967ec7..46ad57ff81 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -30,6 +30,12 @@ fn test_launch_authentication() -> Vec { serde_json::to_vec(&SandboxLaunchAuthentication { supervisor: SupervisorAuthBundle { session_id: openshell_core::SandboxSessionId::new(), + runtime_generation: openshell_core::sandbox_generation::SandboxGenerationId::parse( + "generation-1", + ) + .unwrap(), + session_rotation: openshell_core::jwt::SessionRotation::new(1).unwrap(), + predecessor_session_id: None, gateway_token: SecretJwt::parse("gateway.token.value").unwrap(), gateway_expires_at: i64::MAX, sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(),