diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index f080622549..827b61fd81 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -70,9 +70,10 @@ mise run helm:skaffold:run ``` The Skaffold flow builds distinct `gateway`, `sandbox`, and `supervisor` images -and deploys the OpenShell Helm chart. The Kubernetes compute driver places -`openshell-sandbox` in the workload Pod and `openshell-supervisor` in a separate -Pod, then creates their authenticated channel and NetworkPolicy fence. The +and deploys the OpenShell Helm chart. The Kubernetes driver creates a +capability-free workload Pod and a directly managed capability-free supervisor +Pod. One namespace-wide NetworkPolicy denies direct egress from every OpenShell +workload Pod. The `pkiInitJob` hook (a pre-install Job that runs `openshell-gateway generate-certs`) generates mTLS secrets on first install. The default Skaffold values export gateway and Kubernetes-driver traces to the collector service installed by @@ -238,7 +239,7 @@ SPIFFE JWT-SVIDs for dynamic provider token grants: `openshell.local` and adds a `ClusterSPIFFEID` that maps sandbox pod annotations to `spiffe://openshell.local/openshell/sandbox/`. OpenShell mounts the SPIFFE CSI Workload API socket at -`/spiffe-workload-api/spire-agent.sock` into sandbox pods for provider token +`/spiffe-workload-api/spire-agent.sock` only into supervisor Pods for provider token grants. Supervisor-to-gateway authentication remains on the Kubernetes ServiceAccount bootstrap and gateway-minted sandbox JWT path; the selected Kubernetes compute driver validates the projected token before the gateway diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 99220ab55f..aa5a74e5ed 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -406,16 +406,10 @@ jobs: include: - agent_sandbox_api: v1beta1 agent_sandbox_version: v0.5.0 - topology: combined - extra_helm_values: "" + extra_helm_values: deploy/helm/openshell/ci/values-sandbox-runtime.yaml - agent_sandbox_api: v1alpha1 agent_sandbox_version: v0.4.6 - topology: combined - extra_helm_values: "" - - agent_sandbox_api: v1beta1 - agent_sandbox_version: v0.5.0 - topology: sidecar - extra_helm_values: deploy/helm/openshell/ci/values-sidecar.yaml + extra_helm_values: deploy/helm/openshell/ci/values-sandbox-runtime.yaml permissions: actions: read contents: read @@ -423,7 +417,7 @@ jobs: uses: ./.github/workflows/e2e-kubernetes-test.yml with: image-tag: ${{ github.sha }} - job-name: Kubernetes E2E (Rust smoke, ${{ matrix.topology }}, Agent Sandbox ${{ matrix.agent_sandbox_api }}) + job-name: Kubernetes E2E (Rust smoke, Agent Sandbox ${{ matrix.agent_sandbox_api }}) agent-sandbox-version: ${{ matrix.agent_sandbox_version }} extra-helm-values: ${{ matrix.extra_helm_values }} conformance-artifact-prefix: openshell-conformance @@ -440,6 +434,7 @@ jobs: image-tag: ${{ github.sha }} job-name: Kubernetes E2E (workspace managed mode) e2e-task: e2e:kubernetes:workspace-managed + extra-helm-values: deploy/helm/openshell/ci/values-sandbox-runtime.yaml conformance-artifact-prefix: openshell-conformance kubernetes-external-driver-e2e: @@ -456,7 +451,8 @@ jobs: e2e-task: e2e:kubernetes:external-driver gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu external-driver-binary: openshell-driver-kubernetes - cluster-images: supervisor + cluster-images: sandbox supervisor + extra-helm-values: deploy/helm/openshell/ci/values-sandbox-runtime.yaml conformance-artifact-prefix: openshell-conformance kubernetes-workspace-operator-e2e: @@ -471,6 +467,7 @@ jobs: image-tag: ${{ github.sha }} job-name: Kubernetes E2E (workspace operator mode) e2e-task: e2e:kubernetes:workspace-operator + extra-helm-values: deploy/helm/openshell/ci/values-sandbox-runtime.yaml conformance-artifact-prefix: openshell-conformance kubernetes-ha-e2e: diff --git a/Cargo.lock b/Cargo.lock index c35f5f0516..a4c7430b9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4142,13 +4142,17 @@ dependencies = [ "miette", "notify", "openshell-core", + "openshell-isolation-interface", "openshell-otel", "openshell-otel-test-support", "openshell-policy", + "openshell-sandbox-backend", "opentelemetry", "opentelemetry_sdk", "prost", "prost-types", + "rand 0.9.4", + "rcgen", "serde", "serde_json", "temp-env", diff --git a/README.md b/README.md index 70e6e5c09f..3237c169f9 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,8 @@ uv add openshell Deploy the OpenShell gateway into a Kubernetes cluster from the OCI chart published to GHCR: ```bash -helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true ``` See [`deploy/helm/openshell/README.md`](deploy/helm/openshell/README.md) for available versions, dev tag conventions, and configuration. diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 33608b21e9..0889dc3e60 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -258,7 +258,7 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. |---|---|---|---| | Docker | Local development with Docker available. | Capability-free workload container. | Uses `network_mode=none`; a separate capability-free supervisor container mediates egress and access over a private daemon-local Unix socket volume. | | Podman | Existing rootless driver. | Container. | Not converted by this isolation stack. | -| Kubernetes | Cluster deployment through Helm. | Capability-free sandbox Pod. | Uses empty-egress NetworkPolicy, paired-only supervisor ingress, and a separate capability-free supervisor Deployment over mutually authenticated TLS. It requires an enforcing CNI and trusted sandbox namespace. | +| Kubernetes | Cluster deployment through Helm. | Capability-free sandbox Pod. | Uses one namespace-wide empty-egress workload NetworkPolicy and a separate capability-free supervisor Pod over mutually authenticated TLS. It requires an enforcing CNI and trusted sandbox namespace. | | VM | Experimental microVM isolation. | Per-sandbox libkrun or QEMU VM. | The NIC-less guest runs `openshell-sandbox` as PID 1; host `openshell-supervisor` owns gateway networking and reaches the guest over vsock. | | Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | @@ -334,7 +334,7 @@ Drivers deliver the two binaries to separate trust domains: |---|---| | Docker | A digest-pinned daemon-local volume supplies `openshell-sandbox`; the companion image runs `openshell-supervisor`. | | Podman | Existing driver behavior; not converted by this stack. | -| Kubernetes | A non-root init container stages `openshell-sandbox` into a memory volume; the separate Deployment image runs `openshell-supervisor`. | +| Kubernetes | A non-root init container stages `openshell-sandbox` into a memory volume; a directly managed Pod runs `openshell-supervisor`. | | VM | `openshell-sandbox` is embedded in the guest rootfs; a separately digest-checked native `openshell-supervisor` runs on the host. | | Extension | Defined by the out-of-tree driver. | @@ -365,9 +365,9 @@ and explicit `/sandbox` values select `/sandbox`; other paths must already exist without symlink or reserved-mount collisions and must be usable by the resolved identity. Kubernetes and VM use `/sandbox`. -The Kubernetes driver creates the empty-egress workload fence before a -suspended Sandbox CR, then provisions split immutable bootstrap Secrets, the -private runtime Service, and the supervisor Deployment. A +The Kubernetes driver creates the namespace-wide empty-egress workload fence +before a suspended Sandbox CR, then provisions split immutable bootstrap +Secrets, the private runtime Service, and a gated supervisor Pod. A non-root init container stages `openshell-sandbox` and one-use bootstrap files into memory volumes. The workload Pod never mounts supervisor or gateway credentials. The driver removes its scheduling gate only after the companions diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 76bdf13cac..0d008dfbad 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -396,7 +396,7 @@ CA material, and the other sandbox half of the authenticated channel. For Kubernetes, the operator configures a Secret name and key rather than a gateway-host file path. Kubernetes projects that Secret only into the separate -supervisor Deployment. The sandbox Pod never mounts corporate-proxy credentials +supervisor Pod. The sandbox Pod never mounts corporate-proxy credentials or the interception CA private key. The Basic header travels over the plain-TCP connection to the `http://` proxy, diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index 3a3d843f1a..5b498a24d8 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -16,6 +16,8 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false } +openshell-isolation-interface = { path = "../openshell-isolation-interface" } +openshell-sandbox-backend = { path = "../openshell-sandbox-backend" } openshell-otel = { path = "../openshell-otel" } openshell-policy = { path = "../openshell-policy" } @@ -37,6 +39,8 @@ tracing-subscriber = { workspace = true } thiserror = { workspace = true } miette = { workspace = true } notify = "8" +rand = { workspace = true } +rcgen = { workspace = true } [dev-dependencies] openshell-otel-test-support = { path = "../openshell-otel-test-support" } diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index b64bf0c6e4..c24cd9c9dd 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -45,9 +45,52 @@ not a tenant isolation boundary. ## Runtime Model The gateway stores platform state and delegates sandbox workload creation to -this driver. Kubernetes owns scheduling and pod lifecycle. The -`openshell-sandbox` supervisor inside each workload owns agent isolation, -credential injection, policy polling, logs, and the gateway relay. +this driver. Kubernetes owns scheduling and pod lifecycle. The workload Pod +stages the statically linked musl `openshell-sandbox` binary from +`sandbox_runtime_image`, while a directly managed Pod runs the dynamically +linked glibc `openshell-supervisor` from `supervisor_image`. + +The sandbox owns the agent process, applies Landlock and child seccomp filters, +identifies the binary behind each network syscall, and relays mediated streams +to the supervisor. The supervisor authenticates to the gateway with a JWT, +loads policy and provider state, performs destination and L7 authorization, and +opens upstream connections. The workload receives no gateway credential, +provider identity socket, or corporate-proxy credential. + +Both Pods run as the namespace-resolved non-root UID/GID with +`allowPrivilegeEscalation: false`, `capabilities.drop: [ALL]`, and the runtime +default seccomp profile. The sandbox installs a nested seccomp user-notification +filter without requesting a capability in the Pod spec. Startup fails closed +when the runtime blocks the required seccomp or Landlock operations. + +The supervisor Pod has a direct, non-controller owner reference to the Sandbox +resource. This links its garbage-collection lifecycle to the sandbox without +competing with the Agent Sandbox controller for workload-Pod ownership. + +The driver creates one namespace-wide `NetworkPolicy` before it releases any +workload Pod. It selects every OpenShell workload, denies all workload egress, +and permits OpenShell supervisor Pods to reach the sandbox TLS port. The +authenticated Sandbox Protocol binds each connection to the exact sandbox and +supervisor Pod identities. Supervisors have normal egress for gateway, DNS, +and policy-approved upstream connections unless an operator policy restricts +them. Set +`sandbox_runtime.network_policy_enforced = true` only after verifying that the cluster +CNI enforces ingress and egress `NetworkPolicy` for sandbox namespaces. + +Each sandbox generation uses two immutable bootstrap Secrets. A trusted init +container stages the sandbox bootstrap into memory, and the sandbox removes it +before starting untrusted code. The other Secret is mounted only by the +supervisor. The TLS channel binds the namespace, Sandbox CR, workload Pod, +supervisor Pod, and shared network-policy identities. Stop deletes the workload +and supervisor Pods. Start rotates both Secrets and creates a new supervisor +Pod before releasing a new workload Pod. The shared network fence remains for +the lifetime of the namespace. + +Kubernetes policies are additive, and the API does not attest that the CNI +enforces them. Keep sandbox namespaces administrative: untrusted principals +must not create permissive policies, create Pods, read bootstrap Secrets, or +spoof the OpenShell role labels. Exact supervisor-to-sandbox authorization is +still enforced by TLS, JWT claims, session generation, and recorded Pod UIDs. ## Sandbox Resource @@ -95,73 +138,31 @@ mount attaches an existing PVC under `/sandbox`, which skips the default PVC. ## Credentials, TLS, and Relay -The driver injects gateway callback configuration, sandbox identity, TLS client -material, and the supervisor SSH socket path into the workload. The callback -endpoint is required because the sandbox namespace does not identify the -Gateway Service; Helm renders it from the release topology, while standalone -and raw TOML configurations must set it explicitly. Driver-owned values must -override image-provided environment variables. - -Sandbox pods run as `service_account_name` and keep -`automountServiceAccountToken: false`. The only Kubernetes token exposed to the -supervisor is an explicit, audience-bound projected token mounted at -`/var/run/secrets/openshell/token` for the one-shot `IssueSandboxToken` -bootstrap exchange. The Kubernetes driver authenticates that token through the -compute-driver protocol using its own `service_account_name` and workspace-mode -namespace policy; the gateway receives only the verified sandbox ID. - -The gateway uses the supervisor relay for connect, exec, and file sync. Sandbox -pods do not need direct external ingress for SSH. - -The driver forwards the canonical main-process specification to the process -supervisor and sets pod `restartPolicy: Never`. Main-process environment -overrides stay local to that child; the sidecar bootstrap retains the unmodified -provider environment used by later exec, editor, and SFTP sessions. +Both Pods set `automountServiceAccountToken: false`. The supervisor receives an +explicit audience-bound projected token for the one-shot `IssueSandboxToken` +exchange. The driver verifies that token and the gateway returns the +sandbox-scoped JWT used by the supervisor session. The sandbox Pod receives +neither token. + +The gateway uses the supervisor relay for connect, exec, logs, and file sync. +Sandbox Pods do not need direct external ingress for SSH. + +The driver sends the canonical main-process specification only to the +supervisor. The supervisor passes admitted launch state over the protected +channel. Provider environment updates apply to future exec sessions. ## Container Security Context -The default `combined` supervisor topology grants the sandbox agent container -the Linux capabilities the supervisor needs for namespace setup and process, -filesystem, and network policy enforcement. - -The `sidecar` supervisor topology moves pod-level network setup into a root init -container. In the default process/binary-aware mode, the long-lived network -sidecar runs as UID 0 with `allowPrivilegeEscalation: false`, drops default -Linux capabilities, and adds only `SYS_PTRACE` plus `DAC_READ_SEARCH` for -cross-UID workload `/proc` inspection. The agent container also runs as the -resolved sandbox UID/GID with `allowPrivilegeEscalation: false` and -`capabilities.drop: ["ALL"]`. -Set `sidecar.process_binary_aware_network_policy = false` to run the network -sidecar as the configured non-root `sidecar.proxy_uid`, omit the extra `/proc` -inspection capabilities, and enforce endpoint/L7 network policy without -matching `policy.binaries`. -In this mode OpenShell preserves gateway session and SSH behavior, but the -process supervisor does not perform root-to-sandbox privilege dropping or -supervisor identity mount isolation. It still applies Landlock filesystem policy -and child seccomp filters where the kernel/runtime supports them. Network -endpoint and L7 policy remain enforced by the network sidecar, and -sidecar pods use a shared process namespace so the network sidecar can resolve -process/binary identity through `/proc/`. - -Sidecar mode keeps gateway credentials in the network sidecar. The agent -container does not mount the projected service-account token used for sandbox -token bootstrap, does not mount the sandbox client TLS secret, and does not get -gateway callback environment variables. The process supervisor receives policy -and provider environment state from the sidecar over a local control socket in -the shared sidecar state volume. The sidecar accepts only the pre-workload -process-supervisor connection, authenticates its UID/GID/PID with peer -credentials, and removes the listener afterward. SSH relays use a Linux -abstract socket whose peer PID must match that authenticated supervisor. Both -supervisors exit if the control connection closes, coupling their container -restart lifecycle before a new authoritative client can be established. - -The driver uses the shared AppArmor model through `app_armor_profile`. -Supported values are `Unconfined`, `RuntimeDefault`, and -`Localhost/`; an empty or unset value omits -`securityContext.appArmorProfile`. Docker and Podman translate the same values -to OCI security options. Helm deployments default sandbox agent containers to -`Unconfined` because runtime/default AppArmor profiles can block the -supervisor's network namespace mount setup on AppArmor-enabled nodes. +The sandbox, trusted bootstrap init container, and supervisor request no added +Linux capability. They run as the same numeric non-root identity, disable +privilege escalation, drop all capabilities, and inherit `RuntimeDefault` +seccomp. The sandbox and agent must use the same complete UID, GID, and +supplementary-group identity because the capability-free sandbox cannot change +credentials after launch and must inspect its same-identity descendants. + +The workload Pod does not share host network, PID, IPC, or process namespaces. +The driver uses a scheduling gate to inspect the admitted Pod and bind its UID +into the bootstrap claims before kubelet starts it. ## GPU Support diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 01244da066..f29ba79f5b 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -1,9 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -pub use openshell_core::AppArmorProfile; pub use openshell_core::DynamicStringAllowlist as OperatorNamespaceAllowlist; -use openshell_core::{ImagePullPolicy, config}; +use openshell_core::config; use serde::{Deserialize, Deserializer, Serialize}; use std::collections::BTreeMap; #[cfg(test)] @@ -11,88 +10,89 @@ use std::collections::BTreeSet; use std::path::Path; use std::str::FromStr; -/// Default gateway identity used in managed-mode namespace naming. -pub const DEFAULT_GATEWAY_ID: &str = "openshell"; - -/// Default Kubernetes namespace for sandbox resources. -pub const DEFAULT_K8S_NAMESPACE: &str = "openshell"; - -/// Default Kubernetes `ServiceAccount` assigned to sandbox pods. -pub const DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME: &str = "default"; - -/// Default storage size for the workspace PVC. -pub const DEFAULT_WORKSPACE_STORAGE_SIZE: &str = "2Gi"; - -/// Default non-root UID for relaxed Kubernetes network supervisor sidecars. -pub const DEFAULT_PROXY_UID: u32 = 1337; - -/// How the supervisor binary is delivered into sandbox pods. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub enum SupervisorSideloadMethod { - /// Mount the supervisor OCI image directly as a read-only volume - /// (requires Kubernetes >= v1.33 with the `ImageVolume` feature gate, - /// or >= v1.36 where it is GA). - #[default] - ImageVolume, - /// Copy the binary via an init container and emptyDir volume. - /// Works on all Kubernetes versions. - InitContainer, +/// Image pull policies accepted by the Kubernetes API. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum KubernetesImagePullPolicy { + #[serde(alias = "Always")] + Always, + #[serde(alias = "IfNotPresent")] + IfNotPresent, + #[serde(alias = "Never")] + Never, } -impl std::fmt::Display for SupervisorSideloadMethod { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl KubernetesImagePullPolicy { + /// Return the spelling required by Kubernetes Pod specs. + #[must_use] + pub const fn as_kubernetes_str(self) -> &'static str { match self { - Self::ImageVolume => f.write_str("image-volume"), - Self::InitContainer => f.write_str("init-container"), + Self::Always => "Always", + Self::IfNotPresent => "IfNotPresent", + Self::Never => "Never", } } } -impl FromStr for SupervisorSideloadMethod { +impl FromStr for KubernetesImagePullPolicy { type Err = String; - fn from_str(s: &str) -> Result { - match s { - "image-volume" => Ok(Self::ImageVolume), - "init-container" => Ok(Self::InitContainer), + fn from_str(value: &str) -> Result { + match value { + "always" | "Always" => Ok(Self::Always), + "if_not_present" | "IfNotPresent" => Ok(Self::IfNotPresent), + "never" | "Never" => Ok(Self::Never), other => Err(format!( - "unknown supervisor sideload method '{other}'; expected 'image-volume' or 'init-container'" + "invalid Kubernetes image pull policy '{other}'; expected always, if_not_present, or never" )), } } } -/// How the supervisor is arranged inside Kubernetes sandbox pods. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub enum SupervisorTopology { - /// Run networking and process supervision in the agent container. - #[default] - Combined, - /// Run network supervision in a privileged sidecar and process supervision - /// as a low-capability wrapper in the agent container. - Sidecar, +/// Default gateway identity used in managed-mode namespace naming. +pub const DEFAULT_GATEWAY_ID: &str = "openshell"; + +/// Default Kubernetes namespace for sandbox resources. +pub const DEFAULT_K8S_NAMESPACE: &str = "openshell"; + +/// Default Kubernetes `ServiceAccount` assigned to sandbox pods. +pub const DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME: &str = "default"; + +/// Default storage size for the workspace PVC. +pub const DEFAULT_WORKSPACE_STORAGE_SIZE: &str = "2Gi"; + +/// Driver-owned requirements for the Kubernetes sandbox runtime. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct KubernetesSandboxRuntimeConfig { + /// Explicit operator assertion that the cluster CNI enforces + /// `networking.k8s.io/v1` `NetworkPolicy` for the sandbox namespaces. + pub network_policy_enforced: bool, + /// TCP port exposed by the workload boundary to its paired control pod. + pub boundary_port: u16, } -impl std::fmt::Display for SupervisorTopology { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Combined => f.write_str("combined"), - Self::Sidecar => f.write_str("sidecar"), +impl Default for KubernetesSandboxRuntimeConfig { + fn default() -> Self { + Self { + network_policy_enforced: false, + boundary_port: 5500, } } } -impl FromStr for SupervisorTopology { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "combined" => Ok(Self::Combined), - "sidecar" => Ok(Self::Sidecar), - other => Err(format!("unknown topology '{other}'")), +impl KubernetesSandboxRuntimeConfig { + pub fn validate(&self) -> Result<(), String> { + if !self.network_policy_enforced { + return Err( + "sandbox_runtime.network_policy_enforced must be true after the operator has verified CNI NetworkPolicy enforcement" + .to_string(), + ); + } + if self.boundary_port < 1024 { + return Err("sandbox_runtime.boundary_port must be at least 1024".to_string()); } + Ok(()) } } @@ -138,62 +138,6 @@ impl FromStr for WorkspaceMode { } } -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default, deny_unknown_fields)] -pub struct KubernetesSidecarConfig { - /// UID used by relaxed long-running network sidecars in `sidecar` - /// topology. The network init container installs nftables rules that - /// exempt this UID, so it must not match the sandbox workload UID. - /// Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants - /// the requested `/proc` inspection capabilities into the effective set. - pub proxy_uid: u32, - /// Require process/binary-aware network policy enforcement in sidecar - /// topology. When disabled, the network sidecar runs as `proxy_uid`, - /// drops the extra `/proc` inspection permissions, and evaluates - /// endpoint/L7 policy without matching `policy.binaries`. - pub process_binary_aware_network_policy: bool, -} - -impl Default for KubernetesSidecarConfig { - fn default() -> Self { - Self { - proxy_uid: DEFAULT_PROXY_UID, - process_binary_aware_network_policy: true, - } - } -} - -impl KubernetesSidecarConfig { - pub fn validate_proxy_uid(&self) -> Result<(), String> { - if !(openshell_policy::MIN_SANDBOX_PROXY_UID..=openshell_policy::MAX_SANDBOX_UID) - .contains(&self.proxy_uid) - { - return Err(format!( - "sidecar.proxy_uid must be in range [{}, {}]", - openshell_policy::MIN_SANDBOX_PROXY_UID, - openshell_policy::MAX_SANDBOX_UID, - )); - } - Ok(()) - } -} - -fn deserialize_optional_app_armor_profile<'de, D>( - deserializer: D, -) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let value = Option::::deserialize(deserializer)?; - match value.as_deref() { - None | Some("") => Ok(None), - Some(value) => value - .parse::() - .map(Some) - .map_err(serde::de::Error::custom), - } -} - fn deserialize_provider_spiffe_workload_api_socket_path<'de, D>( deserializer: D, ) -> Result @@ -227,33 +171,33 @@ pub struct KubernetesComputeConfig { /// operator mode. Hot-reloaded on change. Delivered via `ConfigMap` volume mount. #[serde(default, skip_serializing_if = "Option::is_none")] pub operator_namespace_file: Option, - /// Kubernetes `ServiceAccount` assigned to sandbox pods and accepted by - /// the driver's `TokenReview` bootstrap authenticator. + /// Kubernetes `ServiceAccount` assigned to the workload and supervisor Pods. Automatic + /// token mounting is disabled; only the supervisor receives an explicit + /// audience-bound projected token accepted by the bootstrap authenticator. pub service_account_name: String, pub default_image: String, - /// Pull policy for sandbox images. Omit to use Kubernetes's image default. #[serde(default, skip_serializing_if = "Option::is_none")] - pub image_pull_policy: Option, + pub image_pull_policy: Option, /// Kubernetes `imagePullSecrets` names attached to sandbox pods. pub image_pull_secrets: Vec, /// Managed-mode SSH ingress isolation. When enabled, the driver creates a /// `NetworkPolicy` in each managed workspace namespace that permits TCP 2222 /// only from gateway pods matching this peer. pub managed_ssh_ingress: ManagedSshIngressConfig, - /// Image that provides the `openshell-sandbox` supervisor binary. - /// Mounted directly as an image volume, or copied via an init container, - /// depending on `supervisor_sideload_method`. + /// Image that provides the trusted `openshell-sandbox` bootstrap binary. + pub sandbox_runtime_image: String, + /// Kubernetes `imagePullPolicy` for the sandbox runtime image. + /// When omitted, Kubernetes selects its default. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sandbox_runtime_image_pull_policy: Option, + /// Image that provides the trusted `openshell-supervisor` control binary. pub supervisor_image: String, - /// Pull policy for the supervisor image. Omit to use Kubernetes's image - /// default. + /// Kubernetes `imagePullPolicy` for the supervisor image. + /// When omitted, Kubernetes selects its default. #[serde(default, skip_serializing_if = "Option::is_none")] - pub supervisor_image_pull_policy: Option, - /// How the supervisor binary is delivered into sandbox pods. - pub supervisor_sideload_method: SupervisorSideloadMethod, - /// How the supervisor is arranged for Kubernetes sandbox pods. - pub topology: SupervisorTopology, - /// Sidecar-only settings used when `topology = "sidecar"`. - pub sidecar: KubernetesSidecarConfig, + pub supervisor_image_pull_policy: Option, + /// Cross-pod sandbox/supervisor settings. + pub sandbox_runtime: KubernetesSandboxRuntimeConfig, /// Corporate HTTP forward proxy used by the network supervisor for /// policy-approved TLS CONNECT egress. pub https_proxy: Option, @@ -278,14 +222,6 @@ pub struct KubernetesComputeConfig { pub client_tls_secret_name: String, pub host_gateway_ip: String, pub enable_user_namespaces: bool, - /// Kubernetes `AppArmor` profile requested for the sandbox agent container. - /// Empty/None omits the `appArmorProfile` field from sandbox pod specs. - #[serde( - default, - skip_serializing_if = "Option::is_none", - deserialize_with = "deserialize_optional_app_armor_profile" - )] - pub app_armor_profile: Option, pub workspace_default_storage_size: String, /// Kubernetes `StorageClass` name for the default workspace PVC. /// Empty string (default) = omit `storageClassName`, using the cluster's @@ -314,13 +250,10 @@ pub struct KubernetesComputeConfig { deserialize_with = "deserialize_provider_spiffe_workload_api_socket_path" )] pub provider_spiffe_workload_api_socket_path: String, - /// UID used for privilege-drop operations and workspace init container - /// ownership. The supervisor container always runs as UID 0 (root) to - /// create network namespaces and configure Landlock/seccomp; the - /// `sandbox_uid` is injected as the `SANDBOX_UID` environment variable so - /// the supervisor knows which UID to drop to for child processes. + /// Exact UID shared by `openshell-sandbox`, its agent children, the trusted + /// workspace/bootstrap init containers, and `openshell-supervisor`. /// When empty, the driver auto-detects from `OpenShift` SCC annotations on - /// the target namespace; if those are also absent, falls back to `1000`. + /// the target namespace; if those are also absent, falls back to `10001`. #[serde(default, skip_serializing_if = "Option::is_none")] pub sandbox_uid: Option, /// GID used alongside `sandbox_uid` for PVC init container operations. @@ -347,7 +280,7 @@ pub const MAX_SA_TOKEN_TTL_SECS: i64 = 86_400; /// Default sandbox UID used when neither config nor `OpenShift` SCC annotations /// provide a resolved value. -pub(crate) const DEFAULT_SANDBOX_UID: u32 = 1000; +pub(crate) const DEFAULT_SANDBOX_UID: u32 = 10001; /// The annotation key for the `OpenShift` `ServiceAccount` UID range. /// Format: `/` (e.g. `1000000000/10000`). @@ -367,16 +300,18 @@ impl Default for KubernetesComputeConfig { operator_namespace_file: None, service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME.to_string(), default_image: openshell_core::image::default_sandbox_image(), - // Omit the field so Kubernetes applies its own default (Always for - // `latest`, IfNotPresent otherwise). + // Default empty so the gateway omits `imagePullPolicy` from pod + // specs and Kubernetes applies its own default (Always for `latest`, + // IfNotPresent otherwise). `DEFAULT_IMAGE_PULL_POLICY` ("missing") + // is Podman vocabulary and is not a valid Kubernetes value. image_pull_policy: None, image_pull_secrets: Vec::new(), managed_ssh_ingress: ManagedSshIngressConfig::default(), + sandbox_runtime_image: config::default_sandbox_runtime_image(), + sandbox_runtime_image_pull_policy: None, supervisor_image: config::default_supervisor_image(), supervisor_image_pull_policy: None, - supervisor_sideload_method: SupervisorSideloadMethod::default(), - topology: SupervisorTopology::default(), - sidecar: KubernetesSidecarConfig::default(), + sandbox_runtime: KubernetesSandboxRuntimeConfig::default(), https_proxy: None, no_proxy: None, proxy_auth_secret_name: None, @@ -388,7 +323,6 @@ impl Default for KubernetesComputeConfig { client_tls_secret_name: String::new(), host_gateway_ip: String::new(), enable_user_namespaces: false, - app_armor_profile: None, workspace_default_storage_size: DEFAULT_WORKSPACE_STORAGE_SIZE.to_string(), workspace_storage_class: String::new(), default_runtime_class_name: String::new(), @@ -407,7 +341,6 @@ impl KubernetesComputeConfig { self.validate_provider_spiffe_workload_api_socket_path()?; self.validate_sandbox_identity_config()?; self.validate_proxy_uid()?; - self.validate_image_pull_policies()?; self.validate_upstream_proxy_config() } @@ -439,57 +372,20 @@ impl KubernetesComputeConfig { } pub fn validate_proxy_uid(&self) -> Result<(), String> { - self.sidecar.validate_proxy_uid() - } - - /// Reject pull policies Kubernetes cannot express before creating pods. - pub fn validate_image_pull_policies(&self) -> Result<(), String> { - for (field, policy) in [ - ("image_pull_policy", self.image_pull_policy), - ( - "supervisor_image_pull_policy", - self.supervisor_image_pull_policy, - ), - ] { - if policy == Some(ImagePullPolicy::Newer) { - return Err(format!( - "{field} = \"newer\" is supported only by the Podman compute driver" - )); - } - } - Ok(()) - } - - /// Translate a shared policy to Kubernetes's API vocabulary. - pub fn image_pull_policy_value(policy: ImagePullPolicy) -> Result<&'static str, String> { - match policy { - ImagePullPolicy::Always => Ok("Always"), - ImagePullPolicy::IfNotPresent => Ok("IfNotPresent"), - ImagePullPolicy::Never => Ok("Never"), - ImagePullPolicy::Newer => Err( - "image pull policy 'newer' is supported only by the Podman compute driver" - .to_string(), - ), - } + self.sandbox_runtime.validate() } /// Validate the operator-owned corporate upstream proxy configuration. pub fn validate_upstream_proxy_config(&self) -> Result<(), String> { use openshell_core::driver_utils::{UpstreamProxyUrlError, parse_upstream_proxy_url}; - let proxy_addr = self - .https_proxy - .as_deref() - .map(|url| { - parse_upstream_proxy_url(url).map_err(|err| match err { - UpstreamProxyUrlError::Empty => { - "https_proxy must not be empty when set".to_string() - } - UpstreamProxyUrlError::InlineCredentials => "https_proxy must not embed credentials in the URL; supply them through proxy_auth_secret_name and proxy_auth_secret_key".to_string(), - err => format!("https_proxy {err}"), - }) - }) - .transpose()?; + if let Some(url) = &self.https_proxy { + parse_upstream_proxy_url(url).map_err(|err| match err { + UpstreamProxyUrlError::Empty => "https_proxy must not be empty when set".to_string(), + UpstreamProxyUrlError::InlineCredentials => "https_proxy must not embed credentials in the URL; supply them through proxy_auth_secret_name and proxy_auth_secret_key".to_string(), + err => format!("https_proxy {err}"), + })?; + } if let Some(list) = self.no_proxy.as_deref() { if list.trim().is_empty() { @@ -548,17 +444,9 @@ impl KubernetesComputeConfig { .to_string(), ); } - if proxy_addr.as_ref().is_some_and(|proxy| !proxy.secure) - && self.proxy_auth_allow_insecure != Some(true) - { + if self.proxy_auth_allow_insecure != Some(true) { return Err("proxy credentials use cleartext Basic auth over the connection to the http:// proxy; set proxy_auth_allow_insecure = true to accept that exposure, or remove the credential Secret".to_string()); } - if self.topology == SupervisorTopology::Combined { - return Err( - "proxy credential Secrets require topology = \"sidecar\"; combined topology shares the credential mount with the workload and fsGroup can make it readable by the sandbox user" - .to_string(), - ); - } } _ => { return Err( @@ -583,7 +471,7 @@ impl KubernetesComputeConfig { /// 2. `OpenShift` SCC namespace annotations (`sa.scc.uid-range`, /// `sa.scc.supplemental-groups`) — passed in as the optional /// `namespace_annotations` map - /// 3. Fallback defaults: UID=`1000`, GID=UID + /// 3. Fallback defaults: UID=`10001`, GID=UID pub fn resolve_sandbox_uid( &self, namespace_annotations: Option<&BTreeMap>, @@ -870,102 +758,76 @@ mod tests { use std::collections::BTreeMap as HashMap; #[test] - fn default_workspace_storage_size_is_2gi() { - let cfg = KubernetesComputeConfig::default(); - assert_eq!( - cfg.workspace_default_storage_size, - DEFAULT_WORKSPACE_STORAGE_SIZE - ); + fn image_pull_policy_accepts_config_and_kubernetes_spellings() { + for (value, expected) in [ + ("always", KubernetesImagePullPolicy::Always), + ("Always", KubernetesImagePullPolicy::Always), + ("if_not_present", KubernetesImagePullPolicy::IfNotPresent), + ("IfNotPresent", KubernetesImagePullPolicy::IfNotPresent), + ("never", KubernetesImagePullPolicy::Never), + ("Never", KubernetesImagePullPolicy::Never), + ] { + assert_eq!(value.parse(), Ok(expected)); + } + assert!("newer".parse::().is_err()); + assert!("sometimes".parse::().is_err()); } #[test] - fn default_workspace_storage_class_is_empty() { - let cfg = KubernetesComputeConfig::default(); - assert!(cfg.workspace_storage_class.is_empty()); + fn image_pull_policy_fields_reject_unsupported_values() { + for field in [ + "image_pull_policy", + "sandbox_runtime_image_pull_policy", + "supervisor_image_pull_policy", + ] { + let input = format!("{field} = \"sometimes\""); + assert!(toml::from_str::(&input).is_err()); + } } #[test] - fn default_topology_is_combined() { - let cfg = KubernetesComputeConfig::default(); - assert_eq!(cfg.topology, SupervisorTopology::Combined); - assert_eq!(cfg.topology.to_string(), "combined"); + fn published_kubernetes_example_is_valid_toml() { + let docs = include_str!("../../../docs/reference/gateway-config.mdx"); + let section = docs + .split_once("### Kubernetes") + .expect("Kubernetes documentation section") + .1; + let example = section + .split_once("```toml") + .expect("Kubernetes TOML fence") + .1 + .split_once("```") + .expect("closed Kubernetes TOML fence") + .0; + toml::from_str::(example).expect("valid Kubernetes gateway TOML example"); } #[test] - fn default_proxy_uid_is_dedicated_non_root_uid() { + fn default_workspace_storage_size_is_2gi() { let cfg = KubernetesComputeConfig::default(); - assert_eq!(cfg.sidecar.proxy_uid, DEFAULT_PROXY_UID); + assert_eq!( + cfg.workspace_default_storage_size, + DEFAULT_WORKSPACE_STORAGE_SIZE + ); } #[test] - fn default_sidecar_requires_process_binary_aware_network_policy() { + fn default_workspace_storage_class_is_empty() { let cfg = KubernetesComputeConfig::default(); - assert!(cfg.sidecar.process_binary_aware_network_policy); - } - - #[test] - fn image_pull_policy_uses_shared_canonical_values() { - let cfg: KubernetesComputeConfig = serde_json::from_value(serde_json::json!({ - "image_pull_policy": "if_not_present", - "supervisor_image_pull_policy": "never" - })) - .unwrap(); - assert_eq!(cfg.image_pull_policy, Some(ImagePullPolicy::IfNotPresent)); - assert_eq!( - cfg.supervisor_image_pull_policy, - Some(ImagePullPolicy::Never) - ); - - for (policy, expected) in [ - (ImagePullPolicy::Always, "Always"), - (ImagePullPolicy::IfNotPresent, "IfNotPresent"), - (ImagePullPolicy::Never, "Never"), - ] { - assert_eq!( - KubernetesComputeConfig::image_pull_policy_value(policy).unwrap(), - expected - ); - } + assert!(cfg.workspace_storage_class.is_empty()); } #[test] - fn image_pull_policy_rejects_newer_for_sandbox_and_supervisor_images() { + fn sandbox_runtime_requires_network_policy_enforcement_acknowledgement() { + let mut cfg = KubernetesComputeConfig::default(); assert!( - KubernetesComputeConfig::image_pull_policy_value(ImagePullPolicy::Newer) + cfg.validate_proxy_uid() .unwrap_err() - .contains("supported only by the Podman") + .contains("network_policy_enforced") ); - for (sandbox, supervisor) in [ - (Some(ImagePullPolicy::Newer), None), - (None, Some(ImagePullPolicy::Newer)), - ] { - let cfg = KubernetesComputeConfig { - image_pull_policy: sandbox, - supervisor_image_pull_policy: supervisor, - ..KubernetesComputeConfig::default() - }; - let error = cfg.validate_image_pull_policies().unwrap_err(); - assert!(error.contains("supported only by the Podman")); - } - } - - #[test] - fn serde_override_topology_sidecar() { - let json = serde_json::json!({ - "topology": "sidecar" - }); - let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert_eq!(cfg.topology, SupervisorTopology::Sidecar); - } - - #[test] - fn serde_override_topology_combined() { - let json = serde_json::json!({ - "topology": "combined" - }); - let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert_eq!(cfg.topology, SupervisorTopology::Combined); + cfg.sandbox_runtime.network_policy_enforced = true; + cfg.validate_proxy_uid().unwrap(); } #[test] @@ -979,64 +841,6 @@ mod tests { assert!(err.to_string().contains("unknown field")); } - #[test] - fn serde_override_sidecar_process_binary_aware_network_policy_nested() { - let json = serde_json::json!({ - "sidecar": { - "process_binary_aware_network_policy": false - } - }); - let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert!(!cfg.sidecar.process_binary_aware_network_policy); - } - - #[test] - fn serde_override_sidecar_proxy_uid_nested() { - let json = serde_json::json!({ - "sidecar": { - "proxy_uid": 2000 - } - }); - let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert_eq!(cfg.sidecar.proxy_uid, 2000); - cfg.validate_proxy_uid().unwrap(); - } - - #[test] - fn validate_proxy_uid_rejects_privileged_uid() { - let cfg = KubernetesComputeConfig { - sidecar: KubernetesSidecarConfig { - proxy_uid: 999, - ..KubernetesSidecarConfig::default() - }, - ..KubernetesComputeConfig::default() - }; - let err = cfg.validate_proxy_uid().unwrap_err(); - assert!(err.contains("proxy_uid")); - } - - #[test] - fn serde_rejects_invalid_topology() { - let json = serde_json::json!({ - "topology": "unsupported" - }); - let err = serde_json::from_value::(json).unwrap_err(); - assert!(err.to_string().contains("unknown variant")); - } - - #[test] - fn serde_rejects_removed_topology_alias_field() { - let mut json = serde_json::Map::new(); - json.insert( - ["supervisor", "topology"].join("_"), - serde_json::json!("sidecar"), - ); - let err = - serde_json::from_value::(serde_json::Value::Object(json)) - .unwrap_err(); - assert!(err.to_string().contains("unknown field")); - } - #[test] fn serde_rejects_removed_flat_sidecar_fields() { for json in [ @@ -1108,53 +912,6 @@ mod tests { assert!(cfg.default_runtime_class_name.is_empty()); } - #[test] - fn default_app_armor_profile_is_none() { - let cfg = KubernetesComputeConfig::default(); - assert!(cfg.app_armor_profile.is_none()); - } - - #[test] - fn serde_override_app_armor_profile_unconfined() { - let json = serde_json::json!({ - "app_armor_profile": "Unconfined" - }); - let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert_eq!(cfg.app_armor_profile, Some(AppArmorProfile::Unconfined)); - } - - #[test] - fn serde_override_app_armor_profile_runtime_default() { - let json = serde_json::json!({ - "app_armor_profile": "RuntimeDefault" - }); - let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert_eq!(cfg.app_armor_profile, Some(AppArmorProfile::RuntimeDefault)); - } - - #[test] - fn serde_override_app_armor_profile_localhost() { - let json = serde_json::json!({ - "app_armor_profile": "Localhost/openshell-supervisor" - }); - let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert_eq!( - cfg.app_armor_profile, - Some(AppArmorProfile::Localhost( - "openshell-supervisor".to_string() - )) - ); - } - - #[test] - fn serde_empty_app_armor_profile_disables_field() { - let json = serde_json::json!({ - "app_armor_profile": "" - }); - let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert_eq!(cfg.app_armor_profile, None); - } - #[test] fn serde_accepts_absolute_provider_spiffe_socket_path() { let json = serde_json::json!({ @@ -1184,15 +941,6 @@ mod tests { } } - #[test] - fn serde_rejects_invalid_app_armor_profile() { - let json = serde_json::json!({ - "app_armor_profile": "runtime/default" - }); - let err = serde_json::from_value::(json).unwrap_err(); - assert!(err.to_string().contains("unknown AppArmor profile")); - } - #[test] fn serde_override_image_pull_secrets() { let json = serde_json::json!({ @@ -1399,7 +1147,6 @@ mod tests { #[test] fn upstream_proxy_config_accepts_secret_credentials_with_acknowledgement() { let cfg = KubernetesComputeConfig { - topology: SupervisorTopology::Sidecar, https_proxy: Some("http://proxy.corp.example:8080".to_string()), proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()), proxy_auth_secret_key: Some("credentials".to_string()), @@ -1410,22 +1157,9 @@ mod tests { } #[test] - fn upstream_proxy_config_accepts_tls_protected_secret_credentials() { - let cfg = KubernetesComputeConfig { - topology: SupervisorTopology::Sidecar, - https_proxy: Some("https://proxy.corp.example:8443".to_string()), - proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()), - proxy_auth_secret_key: Some("credentials".to_string()), - ..KubernetesComputeConfig::default() - }; - assert!(cfg.validate_upstream_proxy_config().is_ok()); - } - - #[test] - fn toml_deserializes_sidecar_upstream_proxy_settings() { + fn toml_deserializes_upstream_proxy_settings() { let cfg: KubernetesComputeConfig = toml::from_str( r#" - topology = "sidecar" https_proxy = "http://proxy.corp.example:8080" no_proxy = ".svc.cluster.local,10.96.0.0/12" proxy_auth_secret_name = "corporate-proxy-auth" @@ -1547,7 +1281,6 @@ mod tests { "bad key".to_string(), // whitespace is outside the allowed charset ] { let cfg = KubernetesComputeConfig { - topology: SupervisorTopology::Sidecar, https_proxy: Some("http://proxy.corp.example:8080".to_string()), proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()), proxy_auth_secret_key: Some(key.clone()), @@ -1565,7 +1298,6 @@ mod tests { #[test] fn upstream_proxy_config_accepts_max_length_secret_key() { let cfg = KubernetesComputeConfig { - topology: SupervisorTopology::Sidecar, https_proxy: Some("http://proxy.corp.example:8080".to_string()), proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()), proxy_auth_secret_key: Some("a".repeat(253)), @@ -1575,20 +1307,6 @@ mod tests { assert!(cfg.validate_upstream_proxy_config().is_ok()); } - #[test] - fn upstream_proxy_config_rejects_credentials_in_combined_topology() { - let cfg = KubernetesComputeConfig { - topology: SupervisorTopology::Combined, - https_proxy: Some("http://proxy.corp.example:8080".to_string()), - proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()), - proxy_auth_secret_key: Some("credentials".to_string()), - proxy_auth_allow_insecure: Some(true), - ..KubernetesComputeConfig::default() - }; - let err = cfg.validate_upstream_proxy_config().unwrap_err(); - assert!(err.contains("topology = \"sidecar\""), "{err}"); - } - #[test] fn upstream_proxy_config_allows_explicit_false_acknowledgement_without_credentials() { let cfg = KubernetesComputeConfig { diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 3a0f5521ae..6c7a80ea29 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -3,20 +3,28 @@ //! Kubernetes compute driver. -use super::AppArmorProfile; use crate::config::{ - DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, - DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, OperatorNamespaceAllowlist, - SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, is_dns_1123_label, + DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, DEFAULT_WORKSPACE_STORAGE_SIZE, + KubernetesComputeConfig, OperatorNamespaceAllowlist, WorkspaceMode, is_dns_1123_label, managed_namespace, managed_namespace_prefix, validate_managed_namespace_name, }; +use crate::isolation::{ + BOUNDARY_PAIR_LABEL, BOUNDARY_ROLE_LABEL, KubernetesSandboxRuntimeBoundarySpec, +}; +use crate::sandbox_runtime::{ + BOUNDARY_CERTIFICATE_PATH, BOUNDARY_CONFIG_PATH, BOUNDARY_PRIVATE_KEY_PATH, + SANDBOX_SECRET_COMPONENT, SUPERVISOR_SECRET_COMPONENT, SandboxRuntimeNames, boundary_service, + generate_proxy_ca_material, sandbox_bootstrap_secret, + sandbox_owner_reference as sandbox_runtime_sandbox_owner_reference, + supervisor_bootstrap_secret, supervisor_pod, workload_fence, +}; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::authentication::v1::{ TokenReview, TokenReviewSpec, TokenReviewStatus, UserInfo, }; use k8s_openapi::api::core::v1::{ Event as KubeEventObj, Namespace, Node, PersistentVolumeClaimVolumeSource, Pod, Secret, - ServiceAccount, Volume, VolumeMount, + Service, ServiceAccount, Volume, VolumeMount, }; use k8s_openapi::api::networking::v1::{ NetworkPolicy, NetworkPolicyIngressRule, NetworkPolicyPeer, NetworkPolicyPort, @@ -30,13 +38,13 @@ use kube::api::{ use kube::core::gvk::GroupVersionKind; use kube::core::{DynamicObject, ObjectMeta}; use kube::runtime::WatchStreamExt; +use kube::runtime::wait::await_condition; use kube::runtime::watcher::{self, Event}; use kube::{Client, Error as KubeError}; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ LABEL_GATEWAY_ID, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, - LABEL_SANDBOX_NAME, LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, - openshell_sandbox_label_selector, + LABEL_SANDBOX_NAME, LABEL_SANDBOX_WORKSPACE, openshell_sandbox_label_selector, }; use openshell_core::gpu::{driver_gpu_requirements, effective_driver_gpu_count}; use openshell_core::progress::{ @@ -53,8 +61,14 @@ use openshell_core::proto::compute::v1::{ WatchSandboxesSandboxEvent, watch_sandboxes_event, }; use openshell_core::proto_struct::{struct_to_json_object, value_to_json}; +use openshell_isolation_interface::contract::ResolvedWorkloadIdentity; +use openshell_sandbox_backend::boundary_protocol::{ + GatewayVerificationKey, SandboxTlsClientConfig, SandboxTlsServerConfig, + generate_sandbox_tls_material, +}; +use rand::RngCore as _; use serde::Deserialize; -use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::collections::{BTreeMap, HashSet}; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; @@ -68,6 +82,64 @@ pub type WatchStream = const MANAGED_SSH_NETWORK_POLICY_NAME: &str = "openshell-sandbox-ssh"; const AGENT_SANDBOX_TRACE_CONTEXT_ANNOTATION: &str = "opentelemetry.io/trace-context"; +const ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING: &str = "openshell.ai/sandbox-runtime-bootstrapping"; +const ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT: &str = + "openshell.ai/sandbox-runtime-bootstrap-started-at-ms"; +const ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION: &str = + "openshell.ai/sandbox-runtime-bootstrap-operation"; +const ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE: &str = + "openshell.ai/sandbox-runtime-bootstrap-phase"; +const ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_SESSION: &str = + "openshell.ai/sandbox-runtime-bootstrap-session"; +const ANNOTATION_SANDBOX_RUNTIME_GENERATION: &str = "openshell.ai/sandbox-runtime-generation"; +const ANNOTATION_SANDBOX_RUNTIME_READINESS: &str = "openshell.ai/sandbox-runtime-readiness"; +const ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID: &str = "openshell.ai/sandbox-runtime-workload-uid"; +const ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: &str = + "openshell.ai/sandbox-runtime-supervisor-uid"; +const ANNOTATION_SANDBOX_RUNTIME_MAIN_PROCESS_SPEC: &str = + "openshell.ai/sandbox-runtime-main-process-spec"; +const ANNOTATION_SANDBOX_RUNTIME_LOG_LEVEL: &str = "openshell.ai/sandbox-runtime-log-level"; +const ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID: &str = + "openshell.ai/sandbox-runtime-network-policy-uid"; +const ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_VERSION: &str = + "openshell.ai/sandbox-runtime-network-policy-version"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum SandboxRuntimeBootstrapPhase { + Preparing, + Released, + RollingBack, +} + +impl SandboxRuntimeBootstrapPhase { + const fn as_str(self) -> &'static str { + match self { + Self::Preparing => "preparing", + Self::Released => "released", + Self::RollingBack => "rolling-back", + } + } + + fn parse(value: &str) -> Option { + match value { + "preparing" => Some(Self::Preparing), + "released" => Some(Self::Released), + "rolling-back" => Some(Self::RollingBack), + _ => None, + } + } +} + +fn boundary_service_authority( + namespace: &str, + names: &SandboxRuntimeNames, + boundary_port: u16, +) -> String { + format!( + "{}.{}.svc:{boundary_port}", + names.boundary_service, namespace + ) +} #[derive(Debug, thiserror::Error)] pub enum KubernetesDriverError { @@ -86,12 +158,56 @@ pub enum KubernetesDriverError { impl KubernetesDriverError { fn from_kube(err: KubeError) -> Self { match err { - KubeError::Api(api) if api.code == 409 => Self::AlreadyExists, + KubeError::Api(api) if api.code == 409 && api.reason == "AlreadyExists" => { + Self::AlreadyExists + } + KubeError::Api(api) if api.code == 404 => Self::NotFound, other => Self::Message(other.to_string()), } } } +fn is_kube_resource_version_conflict(error: &KubeError) -> bool { + matches!(error, KubeError::Api(api) if api.code == 409 && api.reason == "Conflict") +} + +async fn patch_dynamic_object_with_resource_version_retry( + api: &Api, + name: &str, + mut patch_for_resource_version: impl FnMut(&str) -> serde_json::Value, +) -> Result { + let operation = async { + loop { + let current = api.get(name).await?; + let resource_version = current + .metadata + .resource_version + .as_deref() + .unwrap_or_default(); + let patch = patch_for_resource_version(resource_version); + match api + .patch(name, &PatchParams::default(), &Patch::Merge(&patch)) + .await + { + Ok(updated) => return Ok(updated), + Err(error) if is_kube_resource_version_conflict(&error) => { + tokio::task::yield_now().await; + } + Err(error) => return Err(error), + } + } + }; + tokio::time::timeout(KUBE_API_TIMEOUT, operation) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s updating Kubernetes resource {name}", + KUBE_API_TIMEOUT.as_secs() + )) + })? + .map_err(KubernetesDriverError::from_kube) +} + impl From for openshell_core::ComputeDriverError { fn from(err: KubernetesDriverError) -> Self { match err { @@ -108,6 +224,58 @@ impl From for openshell_core::ComputeDriverError { /// This prevents gRPC handlers from blocking indefinitely when the k8s /// API server is unreachable or slow. const KUBE_API_TIMEOUT: Duration = Duration::from_secs(30); +const SANDBOX_RUNTIME_RECONCILE_INTERVAL: Duration = Duration::from_secs(30); +/// Bound how long a crash-interrupted, fail-closed bootstrap may remain stranded. +const SANDBOX_RUNTIME_BOOTSTRAP_GRACE: Duration = Duration::from_mins(5); + +fn random_sandbox_runtime_token() -> String { + use std::fmt::Write as _; + + let mut bytes = [0_u8; 32]; + rand::rng().fill_bytes(&mut bytes); + let mut token = String::with_capacity(bytes.len() * 2); + for byte in bytes { + write!(token, "{byte:02x}").expect("writing to String cannot fail"); + } + token +} + +fn decode_launch_authentication( + encoded: &[u8], +) -> Result { + let authentication = + serde_json::from_slice::(encoded) + .map_err(|error| { + KubernetesDriverError::Precondition(format!( + "decode Kubernetes sandbox launch authentication: {error}" + )) + })?; + authentication.validate().map_err(|error| { + KubernetesDriverError::Precondition(format!( + "validate Kubernetes sandbox launch authentication: {error}" + )) + })?; + Ok(authentication) +} + +fn gateway_verification_keys( + keys: &[openshell_core::jwt::SessionVerificationKey], +) -> Result, KubernetesDriverError> { + keys.iter() + .map(|key| { + String::from_utf8(key.public_key_pem.clone()) + .map(|public_key_pem| GatewayVerificationKey { + key_id: key.key_id.clone(), + public_key_pem, + }) + .map_err(|error| { + KubernetesDriverError::Precondition(format!( + "Kubernetes sandbox verification key is not UTF-8 PEM: {error}" + )) + }) + }) + .collect() +} /// Kubernetes defaults pod termination to 30 seconds when the pod template /// omits `terminationGracePeriodSeconds`. @@ -289,7 +457,11 @@ const KUBERNETES_DRIVER_RESERVED_VOLUME_NAMES: &[&str] = &[ UPSTREAM_PROXY_AUTH_VOLUME_NAME, SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, SPIFFE_WORKLOAD_API_VOLUME_NAME, - SUPERVISOR_VOLUME_NAME, + SANDBOX_RUNTIME_VOLUME_NAME, + SANDBOX_STATE_VOLUME_NAME, + SANDBOX_BOOTSTRAP_VOLUME_NAME, + SANDBOX_POD_IDENTITY_VOLUME_NAME, + SANDBOX_PROXY_CA_VOLUME_NAME, WORKSPACE_VOLUME_NAME, ]; @@ -447,7 +619,7 @@ const WORKSPACE_MOUNT_PATH: &str = "/sandbox"; /// Mount path for the workspace PVC in the **init** container. A temporary /// path so the init container can see the image's original `/sandbox` and /// copy it into the PVC. -const WORKSPACE_INIT_MOUNT_PATH: &str = "/workspace-pvc"; +const WORKSPACE_INIT_MOUNT_PATH: &str = "/mnt/openshell-workspace"; /// Name of the init container that seeds the workspace PVC. const WORKSPACE_INIT_CONTAINER_NAME: &str = "workspace-init"; @@ -498,7 +670,19 @@ impl KubernetesComputeDriver { shutdown_rx: tokio::sync::watch::Receiver, ) -> Result { config - .validate_configuration() + .validate_workspace_mode() + .map_err(KubernetesDriverError::Precondition)?; + config + .validate_provider_spiffe_workload_api_socket_path() + .map_err(KubernetesDriverError::Precondition)?; + config + .validate_sandbox_identity_config() + .map_err(KubernetesDriverError::Precondition)?; + config + .validate_proxy_uid() + .map_err(KubernetesDriverError::Precondition)?; + config + .validate_upstream_proxy_config() .map_err(KubernetesDriverError::Precondition)?; let base_config = match kube::Config::incluster() { Ok(c) => c, @@ -624,7 +808,7 @@ impl KubernetesComputeDriver { })?; validate_pod_uid(&pod, &identity.pod_uid)?; let sandbox_id = pod_sandbox_id(&pod)?; - let owner = sandbox_owner_reference(&pod)?; + let (owner, via_proxy_control) = Self::resolve_sandbox_owner(&pod, &sandbox_id)?; let sandboxes = self .supported_agent_sandbox_api(self.client.clone(), &identity.namespace) .await @@ -635,10 +819,35 @@ impl KubernetesComputeDriver { warn!(sandbox = %owner.name, %error, "failed to read authenticated Sandbox resource"); tonic::Status::internal("failed to read authenticated Sandbox resource") })?.ok_or_else(|| tonic::Status::permission_denied("sandbox owner not found"))?; - validate_sandbox_owner_identity(owner, &sandbox_id, &sandbox)?; + validate_sandbox_owner_identity(&owner, &sandbox_id, &sandbox)?; + require_proxy_control_authentication(via_proxy_control)?; Ok(sandbox_id) } + #[allow(clippy::result_large_err)] + fn resolve_sandbox_owner( + pod: &Pod, + sandbox_id: &str, + ) -> Result<(OwnerReference, bool), tonic::Status> { + let via_supervisor = pod + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get(BOUNDARY_ROLE_LABEL)) + .is_some_and(|role| role == "supervisor"); + if via_supervisor { + validate_proxy_control_labels(pod, sandbox_id)?; + let names = SandboxRuntimeNames::new(sandbox_id); + if pod.metadata.name.as_deref() != Some(names.supervisor_pod.as_str()) { + return Err(tonic::Status::permission_denied( + "supervisor pod name does not match the sandbox runtime", + )); + } + } + let owner = sandbox_owner_reference(pod, !via_supervisor)?.clone(); + Ok((owner, via_supervisor)) + } + fn accepts_auth_namespace(&self, namespace: &str) -> bool { accepts_auth_namespace(&self.config, self.operator_allowlist.as_ref(), namespace) } @@ -1091,17 +1300,9 @@ impl KubernetesComputeDriver { } fn validate_driver_config_for_sandbox( - &self, sandbox: &Sandbox, ) -> Result { - kubernetes_driver_config_for_spec( - sandbox.spec.as_ref(), - self.config.provider_spiffe_enabled().then_some( - self.config - .provider_spiffe_workload_api_socket_path - .as_str(), - ), - ) + kubernetes_driver_config_for_spec(sandbox.spec.as_ref()) } fn agent_sandbox_api( @@ -1286,8 +1487,7 @@ impl KubernetesComputeDriver { } pub async fn validate_sandbox_create(&self, sandbox: &Sandbox) -> Result<(), tonic::Status> { - let _ = self - .validate_driver_config_for_sandbox(sandbox) + let _ = Self::validate_driver_config_for_sandbox(sandbox) .map_err(tonic::Status::invalid_argument)?; match self.config.workspace_mode { WorkspaceMode::Shared => { @@ -1328,20 +1528,23 @@ impl KubernetesComputeDriver { let selector = self.sandbox_lookup_selector(sandbox_id); let lp = ListParams::default().labels(&selector); match tokio::time::timeout(KUBE_API_TIMEOUT, agent_sandbox_api.api.list(&lp)).await { - Ok(Ok(list)) => list.items.into_iter().next().map_or_else( - || { + Ok(Ok(list)) => { + let Some(obj) = list.items.into_iter().next() else { debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes"); - Ok(None) - }, - |obj| { - let ns = obj - .metadata - .namespace - .clone() - .unwrap_or_else(|| self.config.namespace.clone()); - Ok(sandbox_from_object(&ns, obj).ok().map(|(_, s)| s)) - }, - ), + return Ok(None); + }; + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + Ok( + sandbox_from_object_with_sandbox_runtime_readiness(&self.client, &ns, obj) + .await + .ok() + .map(|(_, sandbox)| sandbox), + ) + } Ok(Err(err)) => { warn!( sandbox_id = %sandbox_id, @@ -1383,25 +1586,23 @@ impl KubernetesComputeDriver { .await { Ok(Ok(list)) => { - let mut sandboxes: Vec = list - .items - .into_iter() - .filter_map(|obj| { - let name = obj.metadata.name.clone().unwrap_or_default(); - let ns = obj - .metadata - .namespace - .clone() - .unwrap_or_else(|| self.config.namespace.clone()); - match sandbox_from_object(&ns, obj) { - Ok((_, s)) => Some(s), - Err(err) => { - warn!(object_name = %name, error = %err, "skipping unrecognized Sandbox in list"); - None - } + let mut sandboxes = Vec::new(); + for obj in list.items { + let name = obj.metadata.name.clone().unwrap_or_default(); + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + match sandbox_from_object_with_sandbox_runtime_readiness(&self.client, &ns, obj) + .await + { + Ok((_, sandbox)) => sandboxes.push(sandbox), + Err(err) => { + warn!(object_name = %name, error = %err, "skipping unrecognized Sandbox in list"); } - }) - .collect(); + } + } sandboxes.sort_by(|left, right| { left.name .cmp(&right.name) @@ -1506,65 +1707,76 @@ impl KubernetesComputeDriver { .resolve_sandbox_identity_in_namespace(&target_namespace) .await; - let image_pull_policy = self - .config - .image_pull_policy - .map(KubernetesComputeConfig::image_pull_policy_value) - .transpose() - .map_err(KubernetesDriverError::Precondition)?; - let supervisor_image_pull_policy = self - .config - .supervisor_image_pull_policy - .map(KubernetesComputeConfig::image_pull_policy_value) - .transpose() - .map_err(KubernetesDriverError::Precondition)?; + let generation = random_sandbox_runtime_token(); + let proxy_names = SandboxRuntimeNames::for_generation(&sandbox.id, &generation); + let main_process_spec = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec( + sandbox.spec.as_ref(), + ) + .map_err(|error| { + KubernetesDriverError::InvalidArgument(format!("encode main process spec: {error}")) + })?; + let log_level = openshell_core::driver_utils::sandbox_log_level(sandbox, "info"); let params = SandboxPodParams { default_image: &self.config.default_image, - image_pull_policy, + image_pull_policy: self.config.image_pull_policy, image_pull_secrets: &self.config.image_pull_secrets, - supervisor_image: &self.config.supervisor_image, - supervisor_image_pull_policy, - supervisor_sideload_method: self.config.supervisor_sideload_method, - topology: self.config.topology, - proxy_uid: self.config.sidecar.proxy_uid, - process_binary_aware_network_policy: self - .config - .sidecar - .process_binary_aware_network_policy, - https_proxy: self.config.https_proxy.as_deref(), - no_proxy: self.config.no_proxy.as_deref(), - proxy_auth_secret_name: self.config.proxy_auth_secret_name.as_deref(), - proxy_auth_secret_key: self.config.proxy_auth_secret_key.as_deref(), - proxy_auth_allow_insecure: self.config.proxy_auth_allow_insecure == Some(true), - proxy_connect_by_hostname: self.config.proxy_connect_by_hostname == Some(true), + sandbox_runtime_image: &self.config.sandbox_runtime_image, + sandbox_runtime_image_pull_policy: self.config.sandbox_runtime_image_pull_policy, service_account_name: &self.config.service_account_name, sandbox_id: &sandbox.id, - sandbox_name: &sandbox.name, - grpc_endpoint: &self.config.grpc_endpoint, - ssh_socket_path: self.ssh_socket_path(), - client_tls_secret_name: &self.config.client_tls_secret_name, - host_gateway_ip: &self.config.host_gateway_ip, enable_user_namespaces: self.config.enable_user_namespaces, - app_armor_profile: self.config.app_armor_profile.as_ref(), workspace_default_storage_size: &self.config.workspace_default_storage_size, workspace_storage_class: &self.config.workspace_storage_class, default_runtime_class_name: &self.config.default_runtime_class_name, - sa_token_ttl_secs: self.config.effective_sa_token_ttl_secs(), - provider_spiffe_enabled: self.config.provider_spiffe_enabled(), - provider_spiffe_workload_api_socket_path: &self - .config - .provider_spiffe_workload_api_socket_path, sandbox_uid: resolved_user_id, sandbox_gid: resolved_group_id, + boundary_port: self.config.sandbox_runtime.boundary_port, + sandbox_secret_name: &proxy_names.sandbox_secret, }; - validate_sidecar_proxy_identity(¶ms)?; - - let data = sandbox_to_k8s_spec(sandbox.spec.as_ref(), ¶ms) - .map_err(KubernetesDriverError::InvalidArgument)?; let kube_name = self.config.kube_resource_name(workspace, name); + let mut data = sandbox_to_k8s_spec(sandbox.spec.as_ref(), ¶ms) + .map_err(KubernetesDriverError::InvalidArgument)?; + self.create_sandbox_runtime_fence(&target_namespace, &proxy_names) + .await?; + // A missing bootstrap Secret keeps both pods inert as defense in + // depth, but the CR is also created suspended so the controller + // never races an unfenced workload into execution. + if agent_sandbox_api.resource.version == SANDBOX_VERSION_V1ALPHA1 { + data["spec"]["replicas"] = serde_json::json!(0); + } else { + data["spec"]["operatingMode"] = serde_json::json!("Suspended"); + } let mut obj = DynamicObject::new(&kube_name, &agent_sandbox_api.resource); let mut annotations = sandbox_annotations(sandbox); add_trace_context_annotation(&mut annotations); + annotations.insert( + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING.to_string(), + "true".to_string(), + ); + annotations.insert( + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT.to_string(), + openshell_core::time::now_ms().to_string(), + ); + annotations.insert( + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION.to_string(), + "create".to_string(), + ); + annotations.insert( + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE.to_string(), + SandboxRuntimeBootstrapPhase::Preparing.as_str().to_string(), + ); + annotations.insert( + ANNOTATION_SANDBOX_RUNTIME_GENERATION.to_string(), + generation.clone(), + ); + annotations.insert( + ANNOTATION_SANDBOX_RUNTIME_MAIN_PROCESS_SPEC.to_string(), + main_process_spec.clone(), + ); + annotations.insert( + ANNOTATION_SANDBOX_RUNTIME_LOG_LEVEL.to_string(), + log_level.clone(), + ); for key in [ crate::config::ANNOTATION_SCC_UID_RANGE, crate::config::ANNOTATION_SCC_SUPPLEMENTAL_GROUPS, @@ -1574,27 +1786,27 @@ impl KubernetesComputeDriver { } } obj.metadata = ObjectMeta { - name: Some(kube_name), - namespace: Some(target_namespace), + name: Some(kube_name.clone()), + namespace: Some(target_namespace.clone()), labels: Some(sandbox_labels(sandbox, Some(&self.config.gateway_id))), annotations: Some(annotations), ..Default::default() }; obj.data = data; - match tokio::time::timeout( + let created = match tokio::time::timeout( KUBE_API_TIMEOUT, agent_sandbox_api.api.create(&PostParams::default(), &obj), ) .await { - Ok(Ok(_result)) => { + Ok(Ok(result)) => { info!( sandbox_id = %sandbox.id, sandbox_name = %name, "Sandbox created in Kubernetes successfully" ); - Ok(()) + result } Ok(Err(err)) => { warn!( @@ -1603,7 +1815,7 @@ impl KubernetesComputeDriver { error = %err, "Failed to create sandbox in Kubernetes" ); - Err(KubernetesDriverError::from_kube(err)) + return Err(KubernetesDriverError::from_kube(err)); } Err(_elapsed) => { warn!( @@ -1612,165 +1824,1232 @@ impl KubernetesComputeDriver { timeout_secs = KUBE_API_TIMEOUT.as_secs(), "Timed out creating sandbox in Kubernetes" ); - Err(KubernetesDriverError::Message(format!( + return Err(KubernetesDriverError::Message(format!( "timed out after {}s waiting for Kubernetes API", KUBE_API_TIMEOUT.as_secs() - ))) + ))); } + }; + if let Err(error) = self + .create_sandbox_runtime_companions( + sandbox, + &target_namespace, + &kube_name, + &agent_sandbox_api, + &created, + &proxy_names, + &generation, + resolved_user_id, + resolved_group_id, + &main_process_spec, + &log_level, + ) + .await + { + warn!(sandbox_id = %sandbox.id, %error, "sandbox-runtime provisioning failed; rolling back Sandbox CR"); + let _ = agent_sandbox_api + .api + .delete(&kube_name, &DeleteParams::default()) + .await; + return Err(error); } + Ok(()) } - #[tracing::instrument( - name = "kubernetes.stop_sandbox", - skip(self), - fields( - otel.name = "kubernetes.stop_sandbox", - otel.status_code = tracing::field::Empty, - sandbox.id = %sandbox_id, - ) - )] - pub async fn stop_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { - let span_status = openshell_otel::ErrorStatusGuard::current(); - let result = self.stop_sandbox_inner(sandbox_id).await; - span_status.finish(result) + async fn create_sandbox_runtime_fence( + &self, + namespace: &str, + names: &SandboxRuntimeNames, + ) -> Result<(), KubernetesDriverError> { + let fence = workload_fence(namespace, names, self.config.sandbox_runtime.boundary_port); + let policies: Api = Api::namespaced(self.client.clone(), namespace); + for (mut policy, component) in [ + (fence.workload_policy, "sandbox-workload-fence"), + (fence.supervisor_policy, "sandbox-supervisor-egress"), + ] { + let labels = policy.metadata.labels.get_or_insert_default(); + labels.insert( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ); + labels.insert("openshell.ai/component".to_string(), component.to_string()); + create_or_validate_sandbox_runtime_fence(&policies, &policy).await?; + } + Ok(()) } - async fn stop_sandbox_inner(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { - let (agent_sandbox_api, kube_name, pod_name, namespace, stop_timeout) = self - .patch_sandbox_operating_state(sandbox_id, false) - .await?; - let pod_api = Api::::namespaced(self.client.clone(), &namespace); - - let deadline = tokio::time::Instant::now() + stop_timeout; - let mut poll_interval = STOP_INITIAL_POLL_INTERVAL; + async fn wait_for_bootstrap_workload_pod( + &self, + pods: &Api, + pod_name: &str, + sandbox_uid: &str, + ) -> Result { + let deadline = tokio::time::Instant::now() + KUBE_API_TIMEOUT; loop { - let now = tokio::time::Instant::now(); - if now >= deadline { - return Err(KubernetesDriverError::Message(format!( - "timed out after {}s waiting for Kubernetes sandbox to stop", - stop_timeout.as_secs() - ))); - } - let request_timeout = KUBE_API_TIMEOUT.min(deadline.saturating_duration_since(now)); - let object = tokio::time::timeout( - request_timeout, - agent_sandbox_api.api.get(&kube_name), - ) - .await - .map_err(|_| { - KubernetesDriverError::Message(format!( - "timed out after {}s waiting for Kubernetes API while checking sandbox stop", - request_timeout.as_secs() - )) - })? - .map_err(KubernetesDriverError::from_kube)?; - if let Some(error) = kubernetes_sandbox_stop_failure(&object) { - return Err(KubernetesDriverError::Message(error)); - } - let pod_is_gone = kubernetes_sandbox_pod_is_gone(&pod_api, &pod_name, deadline) - .await - .map_err(KubernetesDriverError::Message)?; - let stop_is_complete = kubernetes_sandbox_stop_is_complete( - &agent_sandbox_api.resource.version, - &object, - pod_is_gone, - ); - if stop_is_complete { - return Ok(()); - } - let now = tokio::time::Instant::now(); - if now >= deadline { - return Err(KubernetesDriverError::Message(format!( - "timed out after {}s waiting for Kubernetes sandbox to stop", - stop_timeout.as_secs() - ))); + match pods.get_opt(pod_name).await { + Ok(Some(pod)) => { + let owned = pod + .metadata + .owner_references + .as_deref() + .unwrap_or_default() + .iter() + .any(|owner| { + owner.controller == Some(true) + && owner.kind == SANDBOX_KIND + && owner.uid == sandbox_uid + }); + if !owned { + return Err(KubernetesDriverError::Precondition(format!( + "workload Pod {pod_name} is not controlled by the created Sandbox UID" + ))); + } + return Ok(pod); + } + Ok(None) if tokio::time::Instant::now() < deadline => { + tokio::time::sleep(Duration::from_millis(100)).await; + } + Ok(None) => { + return Err(KubernetesDriverError::Message(format!( + "timed out waiting for gated workload Pod {pod_name}" + ))); + } + Err(error) => return Err(KubernetesDriverError::from_kube(error)), } - tokio::time::sleep(poll_interval.min(deadline.saturating_duration_since(now))).await; - poll_interval = next_stop_poll_interval(poll_interval); } } - #[tracing::instrument( - name = "kubernetes.start_sandbox", - skip(self), - fields( - otel.name = "kubernetes.start_sandbox", - otel.status_code = tracing::field::Empty, - sandbox.id = %sandbox_id, - ) - )] - pub async fn start_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { - let span_status = openshell_otel::ErrorStatusGuard::current(); - let result = self - .patch_sandbox_operating_state(sandbox_id, true) - .await - .map(|_| ()); - span_status.finish(result) - } - - async fn patch_sandbox_operating_state( - &self, + #[allow(clippy::too_many_arguments)] + fn validate_capability_free_workload_pod( + pod: &Pod, + sandbox_resource_uid: &str, sandbox_id: &str, - running: bool, - ) -> Result<(AgentSandboxApi, String, String, String, Duration), KubernetesDriverError> { - let lookup_api = self - .supported_sandbox_api_for_lookup(self.client.clone()) - .await - .map_err(KubernetesDriverError::Message)?; - let selector = self.sandbox_lookup_selector(sandbox_id); - let list = tokio::time::timeout( - KUBE_API_TIMEOUT, - lookup_api - .api - .list(&ListParams::default().labels(&selector)), - ) - .await - .map_err(|_| { - KubernetesDriverError::Message(format!( - "timed out after {}s waiting for Kubernetes API", - KUBE_API_TIMEOUT.as_secs() + uid: u32, + gid: u32, + sandbox_secret_name: &str, + ) -> Result<(), KubernetesDriverError> { + let fail = |message: &str| { + KubernetesDriverError::Precondition(format!( + "admitted workload Pod does not preserve capability-free isolation: {message}" )) - })? - .map_err(KubernetesDriverError::from_kube)?; - let object = list - .items - .into_iter() - .next() - .ok_or(KubernetesDriverError::NotFound)?; - let namespace = object + }; + if pod.metadata.uid.as_deref().is_none() { + return Err(fail("missing Pod UID")); + } + let owner_matches = pod .metadata - .namespace - .clone() - .unwrap_or_else(|| self.config.namespace.clone()); - let agent_sandbox_api = Self::agent_sandbox_api( - self.client.clone(), - &lookup_api.resource.version, - &namespace, - ); - let stop_timeout = kubernetes_sandbox_stop_timeout(&object); - let kube_name = object.metadata.name.ok_or_else(|| { - KubernetesDriverError::Message("sandbox resource has no name".to_string()) - })?; - let pod_name = object + .owner_references + .as_deref() + .unwrap_or_default() + .iter() + .any(|owner| { + owner.controller == Some(true) + && owner.kind == SANDBOX_KIND + && owner.uid == sandbox_resource_uid + }); + if !owner_matches { + return Err(fail("Sandbox owner UID changed")); + } + let labels = pod .metadata - .annotations + .labels .as_ref() - .and_then(|annotations| annotations.get(SANDBOX_POD_NAME_ANNOTATION)) - .cloned() - .unwrap_or_else(|| kube_name.clone()); - let resource_version = object.metadata.resource_version.unwrap_or_default(); - let desired = sandbox_operating_state_patch( - &agent_sandbox_api.resource.version, - &resource_version, - running, + .ok_or_else(|| fail("missing labels"))?; + let expected_pair = crate::sandbox_runtime::pair_label_value(sandbox_id); + if labels.get(BOUNDARY_ROLE_LABEL).map(String::as_str) != Some("workload") + || labels.get(BOUNDARY_PAIR_LABEL).map(String::as_str) != Some(expected_pair.as_str()) + { + return Err(fail("pair labels changed")); + } + let spec = pod.spec.as_ref().ok_or_else(|| fail("missing Pod spec"))?; + if spec.host_network == Some(true) + || spec.host_pid == Some(true) + || spec.host_ipc == Some(true) + || spec.share_process_namespace == Some(true) + || spec.automount_service_account_token != Some(false) + { + return Err(fail("host namespaces or ServiceAccount token enabled")); + } + let projected_service_account_token = spec + .volumes + .as_deref() + .unwrap_or_default() + .iter() + .filter_map(|volume| volume.projected.as_ref()) + .flat_map(|projected| projected.sources.as_deref().unwrap_or_default()) + .any(|source| source.service_account_token.is_some()); + if projected_service_account_token { + return Err(fail( + "workload Pod must not receive a projected ServiceAccount token", + )); + } + if spec.restart_policy.as_deref() != Some("Never") + || spec.dns_policy.as_deref() != Some("None") + || spec + .dns_config + .as_ref() + .is_none_or(|dns| dns.nameservers.as_deref() != Some(&["127.0.0.53".to_string()])) + { + return Err(fail("restart or DNS posture changed")); + } + if !spec + .scheduling_gates + .as_deref() + .unwrap_or_default() + .iter() + .any(|gate| gate.name == SANDBOX_BOOTSTRAP_SCHEDULING_GATE) + { + return Err(fail("bootstrap scheduling gate missing")); + } + let security = spec + .security_context + .as_ref() + .ok_or_else(|| fail("missing Pod security context"))?; + if security.run_as_user != Some(i64::from(uid)) + || security.run_as_group != Some(i64::from(gid)) + || security.run_as_non_root != Some(true) + || security.fs_group != Some(i64::from(gid)) + || security + .supplemental_groups + .as_deref() + .is_some_and(|groups| !groups.is_empty()) + || security + .seccomp_profile + .as_ref() + .is_none_or(|profile| profile.type_ != "RuntimeDefault") + { + return Err(fail("numeric identity, groups, or seccomp profile changed")); + } + let pod_json = serde_json::to_value(pod) + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + if let Some(policy) = pod_json.pointer("/spec/securityContext/supplementalGroupsPolicy") + && policy != &serde_json::json!("Strict") + { + return Err(fail("supplementalGroupsPolicy is not Strict")); + } + if pod_json + .pointer("/spec/securityContext/supplementalGroupsPolicy") + .is_none() + { + tracing::warn!( + pod = pod.metadata.name.as_deref().unwrap_or(""), + "Kubernetes omitted supplementalGroupsPolicy; exact runtime groups remain enforced by boundary confirmation" + ); + } + let unprivileged_port_sysctl = pod_json + .pointer("/spec/securityContext/sysctls") + .and_then(serde_json::Value::as_array) + .is_some_and(|sysctls| { + sysctls.iter().any(|sysctl| { + sysctl.get("name").and_then(serde_json::Value::as_str) + == Some("net.ipv4.ip_unprivileged_port_start") + && sysctl.get("value").and_then(serde_json::Value::as_str) == Some("0") + }) + }); + if !unprivileged_port_sysctl { + return Err(fail("safe unprivileged-port sysctl changed")); + } + let check_container = |container: &k8s_openapi::api::core::v1::Container, + name: &str| + -> Result<(), KubernetesDriverError> { + let context = container + .security_context + .as_ref() + .ok_or_else(|| fail(&format!("{name} has no security context")))?; + let drops_all = context + .capabilities + .as_ref() + .and_then(|capabilities| capabilities.drop.as_deref()) + .is_some_and(|drops| drops.iter().any(|capability| capability == "ALL")); + let adds_none = context + .capabilities + .as_ref() + .and_then(|capabilities| capabilities.add.as_deref()) + .is_none_or(<[String]>::is_empty); + if context.run_as_user != Some(i64::from(uid)) + || context.run_as_group != Some(i64::from(gid)) + || context.run_as_non_root != Some(true) + || context.allow_privilege_escalation != Some(false) + || !drops_all + || !adds_none + { + return Err(fail(&format!("{name} security context changed"))); + } + Ok(()) + }; + let agent = spec + .containers + .iter() + .find(|container| container.name == "agent") + .ok_or_else(|| fail("agent container missing"))?; + check_container(agent, "agent")?; + let bootstrap = spec + .init_containers + .as_deref() + .unwrap_or_default() + .iter() + .find(|container| container.name == "openshell-sandbox-bootstrap") + .ok_or_else(|| fail("trusted bootstrap init container missing"))?; + check_container(bootstrap, "bootstrap init container")?; + let mounts_volume = |container: &k8s_openapi::api::core::v1::Container, + volume_name: &str| { + container + .volume_mounts + .as_deref() + .unwrap_or_default() + .iter() + .any(|mount| mount.name == volume_name) + }; + if mounts_volume(agent, SANDBOX_BOOTSTRAP_VOLUME_NAME) + || !mounts_volume(bootstrap, SANDBOX_BOOTSTRAP_VOLUME_NAME) + { + return Err(fail( + "bootstrap Secret must be mounted only by the trusted init container", + )); + } + let secret_matches = spec + .volumes + .as_deref() + .unwrap_or_default() + .iter() + .any(|volume| { + volume.name == SANDBOX_BOOTSTRAP_VOLUME_NAME + && volume + .secret + .as_ref() + .and_then(|secret| secret.secret_name.as_deref()) + == Some(sandbox_secret_name) + }); + if !secret_matches { + return Err(fail("generation-specific sandbox Secret changed")); + } + Ok(()) + } + + #[allow(clippy::too_many_arguments, clippy::similar_names)] + async fn create_sandbox_runtime_companions( + &self, + sandbox: &Sandbox, + namespace: &str, + cr_name: &str, + sandbox_api: &AgentSandboxApi, + sandbox_cr: &DynamicObject, + names: &SandboxRuntimeNames, + _generation: &str, + agent_uid: u32, + agent_gid: u32, + main_process_spec: &str, + log_level: &str, + ) -> Result<(), KubernetesDriverError> { + let cr_uid = sandbox_cr.metadata.uid.as_deref().ok_or_else(|| { + KubernetesDriverError::Message("created Sandbox CR has no UID".to_string()) + })?; + let namespace_uid = Api::::all(self.client.clone()) + .get(namespace) + .await + .map_err(KubernetesDriverError::from_kube)? + .metadata + .uid + .ok_or_else(|| { + KubernetesDriverError::Message("sandbox namespace has no UID".to_string()) + })?; + let dependent_owner = sandbox_runtime_sandbox_owner_reference( + cr_name, + cr_uid, + &sandbox_api.resource.api_version, + false, ); - tokio::time::timeout( + let services: Api = Api::namespaced(self.client.clone(), namespace); + let service = services + .create( + &PostParams::default(), + &boundary_service( + namespace, + names, + &sandbox.id, + self.config.sandbox_runtime.boundary_port, + dependent_owner.clone(), + ), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + let service_ip: std::net::IpAddr = service + .spec + .and_then(|spec| spec.cluster_ip) + .filter(|ip| ip != "None") + .ok_or_else(|| { + KubernetesDriverError::Message("boundary Service has no ClusterIP".to_string()) + })? + .parse() + .map_err(|error| { + KubernetesDriverError::Message(format!( + "invalid boundary Service ClusterIP: {error}" + )) + })?; + + let pods: Api = Api::namespaced(self.client.clone(), namespace); + let supervisor = pods + .create( + &PostParams::default(), + &supervisor_pod( + namespace, + names, + &sandbox.id, + &sandbox.name, + &self.config.gateway_id, + &self.config.supervisor_image, + self.config.supervisor_image_pull_policy, + &self.config.service_account_name, + agent_uid, + agent_gid, + &self.config.image_pull_secrets, + &self.config.grpc_endpoint, + &self.config.client_tls_secret_name, + main_process_spec, + log_level, + self.config.effective_sa_token_ttl_secs(), + self.config.https_proxy.as_deref(), + self.config.no_proxy.as_deref(), + self.config + .proxy_auth_secret_name + .as_deref() + .zip(self.config.proxy_auth_secret_key.as_deref()), + self.config.proxy_auth_allow_insecure == Some(true), + self.config.proxy_connect_by_hostname == Some(true), + self.config.provider_spiffe_enabled().then_some( + self.config + .provider_spiffe_workload_api_socket_path + .as_str(), + ), + dependent_owner.clone(), + ) + .map_err(KubernetesDriverError::Message)?, + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + let supervisor_uid = supervisor.metadata.uid.ok_or_else(|| { + KubernetesDriverError::Message("supervisor Pod has no UID".to_string()) + })?; + + let policies: Api = Api::namespaced(self.client.clone(), namespace); + let fence = policies + .get(&names.workload_policy) + .await + .map_err(KubernetesDriverError::from_kube)?; + let fence_uid = fence.metadata.uid.ok_or_else(|| { + KubernetesDriverError::Message("workload NetworkPolicy has no UID".to_string()) + })?; + let fence_resource_version = fence.metadata.resource_version.ok_or_else(|| { + KubernetesDriverError::Message( + "workload NetworkPolicy has no resourceVersion".to_string(), + ) + })?; + + // Release the Sandbox CR only far enough for the controller to create + // the workload Pod. The Pod remains unschedulable because its template + // carries the OpenShell scheduling gate and references a Secret that + // does not exist yet. + patch_dynamic_object_with_resource_version_retry(&sandbox_api.api, cr_name, |version| { + sandbox_operating_state_patch(&sandbox_api.resource.version, version, true) + }) + .await?; + + let workload_pod = self + .wait_for_bootstrap_workload_pod(&pods, cr_name, cr_uid) + .await?; + Self::validate_capability_free_workload_pod( + &workload_pod, + cr_uid, + &sandbox.id, + agent_uid, + agent_gid, + &names.sandbox_secret, + )?; + let workload_pod_uid = + workload_pod.metadata.uid.clone().ok_or_else(|| { + KubernetesDriverError::Message("workload Pod has no UID".to_string()) + })?; + let workload_pod_name = workload_pod.metadata.name.clone().ok_or_else(|| { + KubernetesDriverError::Message("workload Pod has no name".to_string()) + })?; + patch_dynamic_object_with_resource_version_retry(&sandbox_api.api, cr_name, |version| { + serde_json::json!({ + "metadata": { + "resourceVersion": version, + "annotations": { + ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID: workload_pod_uid.clone(), + ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: supervisor_uid.clone(), + ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID: fence_uid.clone(), + ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_VERSION: fence_resource_version.clone(), + } + } + }) + }) + .await?; + + let launch_authentication = sandbox + .spec + .as_ref() + .filter(|spec| !spec.launch_authentication.is_empty()) + .ok_or_else(|| { + KubernetesDriverError::Precondition( + "Kubernetes sandbox launch authentication is required".to_string(), + ) + }) + .and_then(|spec| decode_launch_authentication(&spec.launch_authentication))?; + let mut child_env = sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()) + .map_or_else(std::collections::HashMap::new, |template| { + template.environment.clone() + }); + if let Some(spec) = sandbox.spec.as_ref() { + child_env.extend(spec.environment.clone()); + } + child_env.retain(|name, _| !name.starts_with("OPENSHELL_")); + let host_gateway_ip = self.config.host_gateway_ip.parse().ok(); + let session_id = launch_authentication.supervisor.session_id; + let tls = generate_sandbox_tls_material(session_id) + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let verification_keys = + gateway_verification_keys(&launch_authentication.verification_keys)?; + let proxy_ca = generate_proxy_ca_material().map_err(KubernetesDriverError::Message)?; + let workload_identity = ResolvedWorkloadIdentity::new( + agent_uid, + agent_gid, + Vec::new(), + "kubernetes-config".to_string(), + format!("sandbox:{cr_uid}"), + ) + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let provisioned = KubernetesSandboxRuntimeBoundarySpec { + boundary_id: sandbox.id.clone(), + generation: launch_authentication + .supervisor + .runtime_generation + .to_string(), + session_id, + session_rotation: launch_authentication.supervisor.session_rotation, + gateway_id: launch_authentication.gateway_id, + verification_keys, + namespace_uid, + sandbox_resource_uid: cr_uid.to_string(), + workload_pod_uid: workload_pod_uid.clone(), + workload_pod_uid_path: PathBuf::from(SANDBOX_POD_UID_PATH), + supervisor_pod_uid: supervisor_uid.clone(), + egress_policy_uid: fence_uid.clone(), + egress_policy_resource_version: fence_resource_version.clone(), + boundary_listener: std::net::SocketAddr::new( + if service_ip.is_ipv6() { + std::net::IpAddr::V6(std::net::Ipv6Addr::UNSPECIFIED) + } else { + std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED) + }, + self.config.sandbox_runtime.boundary_port, + ), + control_authority: boundary_service_authority( + namespace, + names, + self.config.sandbox_runtime.boundary_port, + ), + control_address: std::net::SocketAddr::new( + service_ip, + self.config.sandbox_runtime.boundary_port, + ), + sandbox_tls: SandboxTlsServerConfig { + certificate_chain_path: PathBuf::from(BOUNDARY_CERTIFICATE_PATH), + private_key_path: PathBuf::from(BOUNDARY_PRIVATE_KEY_PATH), + }, + supervisor_tls: SandboxTlsClientConfig { + server_name: tls.server_name.clone(), + trust_anchor_pem: tls.trust_anchor_pem.clone(), + }, + host_gateway_ip, + workload_identity, + child_env, + } + .provision(); + let descriptor = provisioned + .runtime_descriptor + .backend_descriptor() + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let sandbox_secret = sandbox_bootstrap_secret( + namespace, + names, + &sandbox.id, + provisioned + .boundary_config + .encode() + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?, + tls.certificate_chain_pem.into_bytes(), + tls.private_key_pem.into_bytes(), + OwnerReference { + api_version: "v1".to_string(), + kind: "Pod".to_string(), + name: workload_pod_name.clone(), + uid: workload_pod_uid, + controller: Some(false), + block_owner_deletion: Some(false), + }, + ); + let supervisor_secret = supervisor_bootstrap_secret( + namespace, + names, + &sandbox.id, + descriptor.payload, + serde_json::to_vec(&launch_authentication.supervisor).map_err(|error| { + KubernetesDriverError::Message(format!("encode supervisor auth bundle: {error}")) + })?, + proxy_ca.certificate_pem.into_bytes(), + proxy_ca.private_key_pem.into_bytes(), + OwnerReference { + api_version: "v1".to_string(), + kind: "Pod".to_string(), + name: names.supervisor_pod.clone(), + uid: supervisor_uid, + controller: Some(false), + block_owner_deletion: Some(false), + }, + ); + let secrets = Api::::namespaced(self.client.clone(), namespace); + secrets + .create(&PostParams::default(), &sandbox_secret) + .await + .map_err(KubernetesDriverError::from_kube)?; + secrets + .create(&PostParams::default(), &supervisor_secret) + .await + .map_err(KubernetesDriverError::from_kube)?; + + pods.patch( + &names.supervisor_pod, + &PatchParams::default(), + &Patch::Merge(&serde_json::json!({"spec": {"schedulingGates": []}})), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + pods.patch( + &workload_pod_name, + &PatchParams::default(), + &Patch::Merge(&serde_json::json!({"spec": {"schedulingGates": []}})), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + patch_dynamic_object_with_resource_version_retry(&sandbox_api.api, cr_name, |version| { + sandbox_runtime_bootstrap_phase_patch(version, SandboxRuntimeBootstrapPhase::Released) + }) + .await?; + spawn_sandbox_runtime_bootstrap_completion( + pods.clone(), + sandbox_api.api.clone(), + names.supervisor_pod.clone(), + cr_name.to_string(), + Some(cr_uid.to_string()), + ); + // Return while the CR remains explicitly bootstrapping. The gateway + // can now commit the sandbox configuration required by a policy-less + // control process without deadlocking behind this driver call. Only + // boundary PID 1 is running at this point; the agent process cannot + // start until control attaches and confirms enforcement. Reconcile + // removes the marker after the supervisor Pod becomes Ready. + Ok(()) + } + + #[allow(clippy::too_many_arguments, clippy::similar_names)] + async fn install_sandbox_runtime_generation( + &self, + namespace: &str, + cr_name: &str, + sandbox_api: &AgentSandboxApi, + sandbox_id: &str, + cr_uid: &str, + names: &SandboxRuntimeNames, + _generation: &str, + supervisor_uid: &str, + agent_uid: u32, + agent_gid: u32, + child_env: std::collections::HashMap, + launch_authentication: &openshell_core::jwt::SandboxLaunchAuthentication, + ) -> Result<(), KubernetesDriverError> { + let namespace_uid = Api::::all(self.client.clone()) + .get(namespace) + .await + .map_err(KubernetesDriverError::from_kube)? + .metadata + .uid + .ok_or_else(|| { + KubernetesDriverError::Message("sandbox namespace has no UID".to_string()) + })?; + let services = Api::::namespaced(self.client.clone(), namespace); + let service_ip: std::net::IpAddr = services + .get(&names.boundary_service) + .await + .map_err(KubernetesDriverError::from_kube)? + .spec + .and_then(|spec| spec.cluster_ip) + .filter(|ip| ip != "None") + .ok_or_else(|| { + KubernetesDriverError::Message("boundary Service has no ClusterIP".to_string()) + })? + .parse() + .map_err(|error| { + KubernetesDriverError::Message(format!( + "invalid boundary Service ClusterIP: {error}" + )) + })?; + let policies = Api::::namespaced(self.client.clone(), namespace); + let fence = policies + .get(&names.workload_policy) + .await + .map_err(KubernetesDriverError::from_kube)?; + let fence_uid = fence.metadata.uid.ok_or_else(|| { + KubernetesDriverError::Message("workload NetworkPolicy has no UID".to_string()) + })?; + let fence_resource_version = fence.metadata.resource_version.ok_or_else(|| { + KubernetesDriverError::Message( + "workload NetworkPolicy has no resourceVersion".to_string(), + ) + })?; + + let pods = Api::::namespaced(self.client.clone(), namespace); + let workload_pod = self + .wait_for_bootstrap_workload_pod(&pods, cr_name, cr_uid) + .await?; + Self::validate_capability_free_workload_pod( + &workload_pod, + cr_uid, + sandbox_id, + agent_uid, + agent_gid, + &names.sandbox_secret, + )?; + let workload_pod_uid = + workload_pod.metadata.uid.clone().ok_or_else(|| { + KubernetesDriverError::Message("workload Pod has no UID".to_string()) + })?; + let workload_pod_name = workload_pod.metadata.name.clone().ok_or_else(|| { + KubernetesDriverError::Message("workload Pod has no name".to_string()) + })?; + patch_dynamic_object_with_resource_version_retry(&sandbox_api.api, cr_name, |version| { + serde_json::json!({ + "metadata": { + "resourceVersion": version, + "annotations": { + ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID: workload_pod_uid.clone(), + ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: supervisor_uid, + ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID: fence_uid.clone(), + ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_VERSION: fence_resource_version.clone(), + } + } + }) + }) + .await?; + + let session_id = launch_authentication.supervisor.session_id; + let tls = generate_sandbox_tls_material(session_id) + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let verification_keys = + gateway_verification_keys(&launch_authentication.verification_keys)?; + let proxy_ca = generate_proxy_ca_material().map_err(KubernetesDriverError::Message)?; + let workload_identity = ResolvedWorkloadIdentity::new( + agent_uid, + agent_gid, + Vec::new(), + "kubernetes-config".to_string(), + format!("sandbox:{cr_uid}"), + ) + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let provisioned = KubernetesSandboxRuntimeBoundarySpec { + boundary_id: sandbox_id.to_string(), + generation: launch_authentication + .supervisor + .runtime_generation + .to_string(), + session_id, + session_rotation: launch_authentication.supervisor.session_rotation, + gateway_id: launch_authentication.gateway_id.clone(), + verification_keys, + namespace_uid, + sandbox_resource_uid: cr_uid.to_string(), + workload_pod_uid: workload_pod_uid.clone(), + workload_pod_uid_path: PathBuf::from(SANDBOX_POD_UID_PATH), + supervisor_pod_uid: supervisor_uid.to_string(), + egress_policy_uid: fence_uid.clone(), + egress_policy_resource_version: fence_resource_version.clone(), + boundary_listener: std::net::SocketAddr::new( + if service_ip.is_ipv6() { + std::net::IpAddr::V6(std::net::Ipv6Addr::UNSPECIFIED) + } else { + std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED) + }, + self.config.sandbox_runtime.boundary_port, + ), + control_authority: boundary_service_authority( + namespace, + names, + self.config.sandbox_runtime.boundary_port, + ), + control_address: std::net::SocketAddr::new( + service_ip, + self.config.sandbox_runtime.boundary_port, + ), + sandbox_tls: SandboxTlsServerConfig { + certificate_chain_path: PathBuf::from(BOUNDARY_CERTIFICATE_PATH), + private_key_path: PathBuf::from(BOUNDARY_PRIVATE_KEY_PATH), + }, + supervisor_tls: SandboxTlsClientConfig { + server_name: tls.server_name.clone(), + trust_anchor_pem: tls.trust_anchor_pem.clone(), + }, + host_gateway_ip: self.config.host_gateway_ip.parse().ok(), + workload_identity, + child_env, + } + .provision(); + let descriptor = provisioned + .runtime_descriptor + .backend_descriptor() + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let sandbox_secret = sandbox_bootstrap_secret( + namespace, + names, + sandbox_id, + provisioned + .boundary_config + .encode() + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?, + tls.certificate_chain_pem.into_bytes(), + tls.private_key_pem.into_bytes(), + OwnerReference { + api_version: "v1".to_string(), + kind: "Pod".to_string(), + name: workload_pod_name.clone(), + uid: workload_pod_uid, + controller: Some(false), + block_owner_deletion: Some(false), + }, + ); + let supervisor_secret = supervisor_bootstrap_secret( + namespace, + names, + sandbox_id, + descriptor.payload, + serde_json::to_vec(&launch_authentication.supervisor).map_err(|error| { + KubernetesDriverError::Message(format!("encode supervisor auth bundle: {error}")) + })?, + proxy_ca.certificate_pem.into_bytes(), + proxy_ca.private_key_pem.into_bytes(), + OwnerReference { + api_version: "v1".to_string(), + kind: "Pod".to_string(), + name: names.supervisor_pod.clone(), + uid: supervisor_uid.to_string(), + controller: Some(false), + block_owner_deletion: Some(false), + }, + ); + let secrets = Api::::namespaced(self.client.clone(), namespace); + secrets + .create(&PostParams::default(), &sandbox_secret) + .await + .map_err(KubernetesDriverError::from_kube)?; + secrets + .create(&PostParams::default(), &supervisor_secret) + .await + .map_err(KubernetesDriverError::from_kube)?; + + pods.patch( + &names.supervisor_pod, + &PatchParams::default(), + &Patch::Merge(&serde_json::json!({"spec": {"schedulingGates": []}})), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + pods.patch( + &workload_pod_name, + &PatchParams::default(), + &Patch::Merge(&serde_json::json!({"spec": {"schedulingGates": []}})), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + patch_dynamic_object_with_resource_version_retry(&sandbox_api.api, cr_name, |version| { + sandbox_runtime_bootstrap_phase_patch(version, SandboxRuntimeBootstrapPhase::Released) + }) + .await?; + spawn_sandbox_runtime_bootstrap_completion( + pods, + sandbox_api.api.clone(), + names.supervisor_pod.clone(), + cr_name.to_string(), + Some(cr_uid.to_string()), + ); + Ok(()) + } + + #[tracing::instrument( + name = "kubernetes.stop_sandbox", + skip(self), + fields( + otel.name = "kubernetes.stop_sandbox", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox_id, + ) + )] + pub async fn stop_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { + let span_status = openshell_otel::ErrorStatusGuard::current(); + let result = Box::pin(self.stop_sandbox_inner(sandbox_id)).await; + span_status.finish(result) + } + + async fn stop_sandbox_inner(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { + let (agent_sandbox_api, kube_name, pod_name, namespace, stop_timeout) = self + .patch_sandbox_operating_state(sandbox_id, false) + .await?; + let pod_api = Api::::namespaced(self.client.clone(), &namespace); + + let deadline = tokio::time::Instant::now() + stop_timeout; + let mut poll_interval = STOP_INITIAL_POLL_INTERVAL; + loop { + let now = tokio::time::Instant::now(); + if now >= deadline { + return Err(KubernetesDriverError::Message(format!( + "timed out after {}s waiting for Kubernetes sandbox to stop", + stop_timeout.as_secs() + ))); + } + let request_timeout = KUBE_API_TIMEOUT.min(deadline.saturating_duration_since(now)); + let object = tokio::time::timeout( + request_timeout, + agent_sandbox_api.api.get(&kube_name), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s waiting for Kubernetes API while checking sandbox stop", + request_timeout.as_secs() + )) + })? + .map_err(KubernetesDriverError::from_kube)?; + if let Some(error) = kubernetes_sandbox_stop_failure(&object) { + return Err(KubernetesDriverError::Message(error)); + } + let pod_is_gone = kubernetes_sandbox_pod_is_gone(&pod_api, &pod_name, deadline) + .await + .map_err(KubernetesDriverError::Message)?; + let stop_is_complete = kubernetes_sandbox_stop_is_complete( + &agent_sandbox_api.resource.version, + &object, + pod_is_gone, + ); + if stop_is_complete { + self.delete_sandbox_runtime_supervisor(sandbox_id, &namespace) + .await?; + patch_dynamic_object_with_resource_version_retry( + &agent_sandbox_api.api, + &kube_name, + sandbox_runtime_rollback_completion_patch, + ) + .await?; + return Ok(()); + } + let now = tokio::time::Instant::now(); + if now >= deadline { + return Err(KubernetesDriverError::Message(format!( + "timed out after {}s waiting for Kubernetes sandbox to stop", + stop_timeout.as_secs() + ))); + } + tokio::time::sleep(poll_interval.min(deadline.saturating_duration_since(now))).await; + poll_interval = next_stop_poll_interval(poll_interval); + } + } + + #[tracing::instrument( + name = "kubernetes.start_sandbox", + skip(self), + fields( + otel.name = "kubernetes.start_sandbox", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox_id, + ) + )] + pub async fn start_sandbox( + &self, + sandbox_id: &str, + generation_id: &str, + launch_authentication: &[u8], + ) -> Result<(), KubernetesDriverError> { + let span_status = openshell_otel::ErrorStatusGuard::current(); + let result = Box::pin(self.start_sandbox_runtime_generation( + sandbox_id, + generation_id, + launch_authentication, + )) + .await; + span_status.finish(result) + } + + #[allow(clippy::similar_names)] + async fn start_sandbox_runtime_generation( + &self, + sandbox_id: &str, + encoded_generation: &str, + encoded_authentication: &[u8], + ) -> Result<(), KubernetesDriverError> { + let generation = openshell_core::sandbox_generation::SandboxGenerationId::parse( + encoded_generation.to_string(), + ) + .map_err(|error| KubernetesDriverError::InvalidArgument(error.to_string()))?; + let launch_authentication = decode_launch_authentication(encoded_authentication)?; + let lookup_api = self + .supported_sandbox_api_for_lookup(self.client.clone()) + .await + .map_err(KubernetesDriverError::Message)?; + let selector = self.sandbox_lookup_selector(sandbox_id); + let mut objects = lookup_api + .api + .list(&ListParams::default().labels(&selector)) + .await + .map_err(KubernetesDriverError::from_kube)? + .items; + let mut object = objects.pop().ok_or(KubernetesDriverError::NotFound)?; + if sandbox_runtime_bootstrap_in_progress(&object) { + let phase = sandbox_runtime_bootstrap_phase(&object); + if phase != Some(SandboxRuntimeBootstrapPhase::RollingBack) + && sandbox_runtime_bootstrap_operation(&object) != Some("restart") + { + return Err(KubernetesDriverError::Precondition( + "initial sandbox bootstrap has not completed; wait for reconciliation" + .to_string(), + )); + } + if sandbox_runtime_generation(&object) == Some(generation.as_str()) + && sandbox_runtime_bootstrap_phase(&object) + == Some(SandboxRuntimeBootstrapPhase::Released) + { + let namespace = object + .metadata + .namespace + .as_deref() + .unwrap_or(&self.config.namespace); + if sandbox_runtime_control_availability(&self.client, namespace, sandbox_id).await + == SandboxRuntimeControlAvailability::Available + && sandbox_runtime_runtime_is_ready(&object) + { + self.complete_sandbox_runtime_bootstrap(&lookup_api, &object) + .await; + return Ok(()); + } + } + + // A retry adopts the durable generation identity, but reconstructs + // its resources from a clean suspended state. This is safe even + // after gateway replacement because launch credentials stay in + // memory and are supplied again by the caller. + self.stop_sandbox_inner(sandbox_id).await?; + let mut refreshed = lookup_api + .api + .list(&ListParams::default().labels(&selector)) + .await + .map_err(KubernetesDriverError::from_kube)? + .items; + object = refreshed.pop().ok_or(KubernetesDriverError::NotFound)?; + } + let namespace = object + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + let cr_name = object.metadata.name.as_deref().ok_or_else(|| { + KubernetesDriverError::Message("sandbox resource has no name".to_string()) + })?; + let cr_uid = object.metadata.uid.as_deref().ok_or_else(|| { + KubernetesDriverError::Message("sandbox resource has no UID".to_string()) + })?; + let sandbox_api = Self::agent_sandbox_api( + self.client.clone(), + &lookup_api.resource.version, + &namespace, + ); + let pods = Api::::namespaced(self.client.clone(), &namespace); + if pods + .get_opt(cr_name) + .await + .map_err(KubernetesDriverError::from_kube)? + .is_some() + { + if !encoded_authentication.is_empty() { + // Gateway restart creates a fresh in-memory launch session. + // Recreate both Pods so neither side retains credentials for + // the gateway instance that was replaced. + self.stop_sandbox_inner(sandbox_id).await?; + return Box::pin(self.start_sandbox_runtime_generation( + sandbox_id, + encoded_generation, + encoded_authentication, + )) + .await; + } + return Err(KubernetesDriverError::Precondition( + "cannot rotate a running workload Pod; stop the sandbox first".to_string(), + )); + } + + let names = SandboxRuntimeNames::for_generation(sandbox_id, generation.as_str()); + self.create_sandbox_runtime_fence(&namespace, &names) + .await?; + self.delete_sandbox_runtime_supervisor(sandbox_id, &namespace) + .await?; + let main_process_spec = + required_sandbox_annotation(&object, ANNOTATION_SANDBOX_RUNTIME_MAIN_PROCESS_SPEC)?; + let log_level = required_sandbox_annotation(&object, ANNOTATION_SANDBOX_RUNTIME_LOG_LEVEL)?; + let sandbox_name = annotation_or_label(&object, LABEL_SANDBOX_NAME).ok_or_else(|| { + KubernetesDriverError::Precondition( + "Sandbox resource is missing its OpenShell sandbox name".to_string(), + ) + })?; + let (agent_uid, agent_gid, _) = + self.resolve_sandbox_identity_in_namespace(&namespace).await; + let supervisor = pods + .create( + &PostParams::default(), + &supervisor_pod( + &namespace, + &names, + sandbox_id, + &sandbox_name, + &self.config.gateway_id, + &self.config.supervisor_image, + self.config.supervisor_image_pull_policy, + &self.config.service_account_name, + agent_uid, + agent_gid, + &self.config.image_pull_secrets, + &self.config.grpc_endpoint, + &self.config.client_tls_secret_name, + &main_process_spec, + &log_level, + self.config.effective_sa_token_ttl_secs(), + self.config.https_proxy.as_deref(), + self.config.no_proxy.as_deref(), + self.config + .proxy_auth_secret_name + .as_deref() + .zip(self.config.proxy_auth_secret_key.as_deref()), + self.config.proxy_auth_allow_insecure == Some(true), + self.config.proxy_connect_by_hostname == Some(true), + self.config.provider_spiffe_enabled().then_some( + self.config + .provider_spiffe_workload_api_socket_path + .as_str(), + ), + sandbox_runtime_sandbox_owner_reference( + cr_name, + cr_uid, + &sandbox_api.resource.api_version, + false, + ), + ) + .map_err(KubernetesDriverError::Message)?, + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + let supervisor_uid = supervisor.metadata.uid.ok_or_else(|| { + KubernetesDriverError::Message("replacement supervisor Pod has no UID".to_string()) + })?; + + let mut volumes = object + .data + .pointer("/spec/podTemplate/spec/volumes") + .and_then(serde_json::Value::as_array) + .cloned() + .ok_or_else(|| { + KubernetesDriverError::Precondition( + "Sandbox pod template has no volume list".to_string(), + ) + })?; + let sandbox_secret = volumes + .iter_mut() + .find(|volume| { + volume.get("name").and_then(serde_json::Value::as_str) + == Some(SANDBOX_BOOTSTRAP_VOLUME_NAME) + }) + .and_then(|volume| volume.get_mut("secret")) + .ok_or_else(|| { + KubernetesDriverError::Precondition( + "Sandbox pod template is missing its bootstrap Secret volume".to_string(), + ) + })?; + sandbox_secret["secretName"] = serde_json::json!(names.sandbox_secret); + let restart = async { + patch_dynamic_object_with_resource_version_retry( + &sandbox_api.api, + cr_name, + |version| { + let mut running_patch = + sandbox_operating_state_patch(&sandbox_api.resource.version, version, true); + running_patch["metadata"]["annotations"] = serde_json::json!({ + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING: "true", + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT: openshell_core::time::now_ms().to_string(), + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION: "restart", + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE: SandboxRuntimeBootstrapPhase::Preparing.as_str(), + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_SESSION: launch_authentication.supervisor.session_id.to_string(), + ANNOTATION_SANDBOX_RUNTIME_GENERATION: generation.as_str(), + ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: supervisor_uid.clone(), + }); + running_patch["spec"]["podTemplate"]["spec"]["volumes"] = + serde_json::Value::Array(volumes.clone()); + running_patch + }, + ) + .await?; + + let child_env = child_environment_from_sandbox_object(&object); + self.install_sandbox_runtime_generation( + &namespace, + cr_name, + &sandbox_api, + sandbox_id, + cr_uid, + &names, + generation.as_str(), + &supervisor_uid, + agent_uid, + agent_gid, + child_env, + &launch_authentication, + ) + .await + } + .await; + + if let Err(error) = restart { + if let Err(rollback_error) = self.stop_sandbox_inner(sandbox_id).await { + return Err(KubernetesDriverError::Message(format!( + "restart failed: {error}; rollback remains pending: {rollback_error}" + ))); + } + return Err(error); + } + Ok(()) + } + + async fn patch_sandbox_operating_state( + &self, + sandbox_id: &str, + running: bool, + ) -> Result<(AgentSandboxApi, String, String, String, Duration), KubernetesDriverError> { + let lookup_api = self + .supported_sandbox_api_for_lookup(self.client.clone()) + .await + .map_err(KubernetesDriverError::Message)?; + let selector = self.sandbox_lookup_selector(sandbox_id); + let list = tokio::time::timeout( KUBE_API_TIMEOUT, - agent_sandbox_api.api.patch( - &kube_name, - &PatchParams::default(), - &Patch::Merge(&desired), - ), + lookup_api + .api + .list(&ListParams::default().labels(&selector)), ) .await .map_err(|_| { @@ -1780,6 +3059,40 @@ impl KubernetesComputeDriver { )) })? .map_err(KubernetesDriverError::from_kube)?; + let object = list + .items + .into_iter() + .next() + .ok_or(KubernetesDriverError::NotFound)?; + let namespace = object + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + let agent_sandbox_api = Self::agent_sandbox_api( + self.client.clone(), + &lookup_api.resource.version, + &namespace, + ); + let stop_timeout = kubernetes_sandbox_stop_timeout(&object); + let kube_name = object.metadata.name.clone().ok_or_else(|| { + KubernetesDriverError::Message("sandbox resource has no name".to_string()) + })?; + let pod_name = object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(SANDBOX_POD_NAME_ANNOTATION)) + .cloned() + .unwrap_or_else(|| kube_name.clone()); + patch_dynamic_object_with_resource_version_retry( + &agent_sandbox_api.api, + &kube_name, + |version| { + sandbox_operating_state_patch(&agent_sandbox_api.resource.version, version, running) + }, + ) + .await?; info!( sandbox_id, @@ -1796,6 +3109,84 @@ impl KubernetesComputeDriver { )) } + async fn delete_sandbox_runtime_supervisor( + &self, + sandbox_id: &str, + namespace: &str, + ) -> Result<(), KubernetesDriverError> { + let names = SandboxRuntimeNames::new(sandbox_id); + let pods = Api::::namespaced(self.client.clone(), namespace); + if let Some(pod) = pods + .get_opt(&names.supervisor_pod) + .await + .map_err(KubernetesDriverError::from_kube)? + { + pods.delete( + &names.supervisor_pod, + &DeleteParams::foreground().preconditions(Preconditions { + uid: pod.metadata.uid, + resource_version: None, + }), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + let deadline = tokio::time::Instant::now() + KUBE_API_TIMEOUT; + loop { + if pods + .get_opt(&names.supervisor_pod) + .await + .map_err(KubernetesDriverError::from_kube)? + .is_none() + { + break; + } + if tokio::time::Instant::now() >= deadline { + return Err(KubernetesDriverError::Message( + "timed out waiting for supervisor Pod deletion".to_string(), + )); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + self.delete_sandbox_runtime_generation_secrets(sandbox_id, namespace) + .await + } + + async fn delete_sandbox_runtime_generation_secrets( + &self, + sandbox_id: &str, + namespace: &str, + ) -> Result<(), KubernetesDriverError> { + let secrets = Api::::namespaced(self.client.clone(), namespace); + for component in [SANDBOX_SECRET_COMPONENT, SUPERVISOR_SECRET_COMPONENT] { + let selector = + format!("openshell.ai/sandbox-id={sandbox_id},openshell.ai/component={component}"); + let items = secrets + .list(&ListParams::default().labels(&selector)) + .await + .map_err(KubernetesDriverError::from_kube)?; + for secret in items { + let Some(name) = secret.metadata.name else { + continue; + }; + match secrets + .delete( + &name, + &DeleteParams::default().preconditions(Preconditions { + uid: secret.metadata.uid, + resource_version: None, + }), + ) + .await + { + Ok(_) | Err(KubeError::Api(kube::core::ErrorResponse { code: 404, .. })) => {} + Err(error) => return Err(KubernetesDriverError::from_kube(error)), + } + } + } + Ok(()) + } + #[tracing::instrument( name = "kubernetes.delete_sandbox", skip(self), @@ -1811,127 +3202,669 @@ impl KubernetesComputeDriver { span_status.finish(result) } - async fn delete_sandbox_inner(&self, sandbox_id: &str) -> Result { - info!( - sandbox_id = %sandbox_id, - workspace_mode = %self.config.workspace_mode, - "Deleting sandbox from Kubernetes" - ); - - let lookup_api = self - .supported_sandbox_api_for_lookup(self.client.clone()) - .await?; - let selector = self.sandbox_lookup_selector(sandbox_id); - let lp = ListParams::default().labels(&selector); - let (kube_name, obj_namespace, _workspace, preconditions) = match tokio::time::timeout( + async fn delete_sandbox_inner(&self, sandbox_id: &str) -> Result { + info!( + sandbox_id = %sandbox_id, + workspace_mode = %self.config.workspace_mode, + "Deleting sandbox from Kubernetes" + ); + + let lookup_api = self + .supported_sandbox_api_for_lookup(self.client.clone()) + .await?; + let selector = self.sandbox_lookup_selector(sandbox_id); + let lp = ListParams::default().labels(&selector); + let (kube_name, obj_namespace, _workspace, preconditions, pod_name) = + match tokio::time::timeout(KUBE_API_TIMEOUT, lookup_api.api.list(&lp)).await { + Ok(Ok(list)) => { + if let Some(obj) = list.items.into_iter().next() { + match obj.metadata.name.clone() { + Some(name) => { + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + let ws = obj + .metadata + .labels + .as_ref() + .and_then(|l| l.get(LABEL_SANDBOX_WORKSPACE).cloned()) + .unwrap_or_default(); + let pc = Preconditions { + uid: obj.metadata.uid, + resource_version: obj.metadata.resource_version, + }; + let pod_name = obj + .metadata + .annotations + .as_ref() + .and_then(|annotations| { + annotations.get(SANDBOX_POD_NAME_ANNOTATION) + }) + .cloned() + .unwrap_or_else(|| name.clone()); + (name, ns, ws, pc, pod_name) + } + None => return Ok(false), + } + } else { + debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted)"); + return Ok(false); + } + } + Ok(Err(err)) => { + warn!( + sandbox_id = %sandbox_id, + error = %err, + "Failed to list sandbox for deletion from Kubernetes" + ); + return Err(err.to_string()); + } + Err(_elapsed) => { + warn!( + sandbox_id = %sandbox_id, + timeout_secs = KUBE_API_TIMEOUT.as_secs(), + "Timed out listing sandbox for deletion from Kubernetes" + ); + return Err(format!( + "timed out after {}s waiting for Kubernetes API", + KUBE_API_TIMEOUT.as_secs() + )); + } + }; + + let delete_api = self + .supported_agent_sandbox_api(self.client.clone(), &obj_namespace) + .await?; + let dp = DeleteParams::default().preconditions(preconditions); + match tokio::time::timeout(KUBE_API_TIMEOUT, delete_api.api.delete(&kube_name, &dp)).await { + Ok(Ok(_response)) => { + info!(sandbox_id = %sandbox_id, namespace = %obj_namespace, "Sandbox deleted from Kubernetes"); + { + let pod_api = Api::::namespaced(self.client.clone(), &obj_namespace); + let deadline = tokio::time::Instant::now() + + DEFAULT_POD_TERMINATION_GRACE_PERIOD + + KUBE_API_TIMEOUT; + loop { + match kubernetes_sandbox_pod_is_gone(&pod_api, &pod_name, deadline).await { + Ok(true) => break, + Ok(false) if tokio::time::Instant::now() < deadline => { + tokio::time::sleep(STOP_INITIAL_POLL_INTERVAL).await; + } + Ok(false) | Err(_) => { + warn!( + sandbox_id, + "retaining sandbox-runtime workload fence because workload Pod deletion was not confirmed" + ); + return Ok(true); + } + } + } + // API acceptance of DELETE does not mean the CR is gone. + // Keep the unowned fence while finalizers can still leave + // the controller able to reconcile a workload Pod. + if !matches!( + tokio::time::timeout(KUBE_API_TIMEOUT, delete_api.api.get(&kube_name)) + .await, + Ok(Err(KubeError::Api(kube::core::ErrorResponse { + code: 404, + .. + }))) + ) { + warn!(sandbox_id, "Sandbox CR deletion was not confirmed"); + return Ok(true); + } + } + Ok(true) + } + Ok(Err(KubeError::Api(err))) if err.code == 404 || err.code == 409 => { + debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted or replaced)"); + Ok(false) + } + Ok(Err(err)) => { + warn!( + sandbox_id = %sandbox_id, + error = %err, + "Failed to delete sandbox from Kubernetes" + ); + Err(err.to_string()) + } + Err(_elapsed) => { + warn!( + sandbox_id = %sandbox_id, + timeout_secs = KUBE_API_TIMEOUT.as_secs(), + "Timed out deleting sandbox from Kubernetes" + ); + Err(format!( + "timed out after {}s waiting for Kubernetes API", + KUBE_API_TIMEOUT.as_secs() + )) + } + } + } + + pub async fn sandbox_exists(&self, sandbox_id: &str) -> Result { + let agent_sandbox_api = self + .supported_sandbox_api_for_lookup(self.client.clone()) + .await?; + let selector = self.sandbox_lookup_selector(sandbox_id); + let lp = ListParams::default().labels(&selector); + match tokio::time::timeout(KUBE_API_TIMEOUT, agent_sandbox_api.api.list(&lp)).await { + Ok(Ok(list)) => Ok(!list.items.is_empty()), + Ok(Err(err)) => Err(err.to_string()), + Err(_elapsed) => Err(format!( + "timed out after {}s waiting for Kubernetes API", + KUBE_API_TIMEOUT.as_secs() + )), + } + } + + /// Repair lifecycle drift and reap unowned workload fences while the + /// gateway's sandbox watch is alive. Bootstrap material is immutable and + /// intentionally not read by the gateway, so this pass only repairs state + /// that can be proven from the Sandbox CR and named companion objects. + async fn reconcile_sandbox_runtime_resources(&self) { + let lookup_api = match self + .supported_sandbox_api_for_lookup(self.client.clone()) + .await + { + Ok(api) => api, + Err(error) => { + warn!(%error, "skipping sandbox-runtime reconciliation: Sandbox API unavailable"); + return; + } + }; + let list = match tokio::time::timeout( + KUBE_API_TIMEOUT, + lookup_api + .api + .list(&ListParams::default().labels(&self.openshell_sandbox_selector())), + ) + .await + { + Ok(Ok(list)) => list, + Ok(Err(error)) => { + warn!(%error, "skipping sandbox-runtime reconciliation: Sandbox list failed"); + return; + } + Err(_) => { + warn!("skipping sandbox-runtime reconciliation: Sandbox list timed out"); + return; + } + }; + + for object in list.items { + let Ok(sandbox_id) = sandbox_id_from_object(&object) else { + continue; + }; + let namespace = object + .metadata + .namespace + .as_deref() + .unwrap_or(&self.config.namespace); + let cr_name = object.metadata.name.as_deref().unwrap_or_default(); + if sandbox_runtime_bootstrap_phase(&object) + == Some(SandboxRuntimeBootstrapPhase::RollingBack) + { + self.reconcile_sandbox_runtime_rollback( + &lookup_api, + &object, + &sandbox_id, + namespace, + cr_name, + ) + .await; + continue; + } + let names = SandboxRuntimeNames::new(&sandbox_id); + let policies = Api::::namespaced(self.client.clone(), namespace); + match self.create_sandbox_runtime_fence(namespace, &names).await { + Ok(()) => {} + Err(KubernetesDriverError::Precondition(error)) => { + warn!(sandbox_id, %error, "sandbox-runtime workload fence is altered; suspending workload"); + self.suspend_sandbox_runtime_after_dependency_failure(&lookup_api, &object) + .await; + continue; + } + Err(error) => { + warn!(sandbox_id, %error, "could not verify sandbox-runtime workload fence; reconciliation will retry"); + continue; + } + } + let Ok(Ok(fence)) = + tokio::time::timeout(KUBE_API_TIMEOUT, policies.get(&names.workload_policy)).await + else { + continue; + }; + if !sandbox_runtime_namespace_fence_generation_matches(&fence, &object) { + warn!( + sandbox_id, + "namespace workload fence generation changed; suspending stale boundary" + ); + self.suspend_sandbox_runtime_after_dependency_failure(&lookup_api, &object) + .await; + continue; + } + if sandbox_runtime_bootstrap_in_progress(&object) { + if sandbox_runtime_control_availability(&self.client, namespace, &sandbox_id).await + == SandboxRuntimeControlAvailability::Available + && sandbox_runtime_runtime_is_ready(&object) + { + self.complete_sandbox_runtime_bootstrap(&lookup_api, &object) + .await; + } else { + self.reap_stale_sandbox_runtime_bootstrap(&lookup_api, &object) + .await; + } + continue; + } + let desired_running = sandbox_runtime_should_run(&object); + if desired_running { + match sandbox_runtime_workload_generation_matches( + &self.client, + namespace, + cr_name, + &object, + ) + .await + { + SandboxRuntimeControlAvailability::Available => {} + SandboxRuntimeControlAvailability::Unavailable => { + warn!( + sandbox_id, + "sandbox-runtime workload generation changed; suspending stale boundary" + ); + self.suspend_sandbox_runtime_after_dependency_failure(&lookup_api, &object) + .await; + continue; + } + SandboxRuntimeControlAvailability::Unknown => { + warn!( + sandbox_id, + "could not verify sandbox-runtime workload generation; reconciliation will retry" + ); + continue; + } + } + match sandbox_runtime_supervisor_generation_matches( + &self.client, + namespace, + &sandbox_id, + &object, + ) + .await + { + SandboxRuntimeControlAvailability::Available => {} + SandboxRuntimeControlAvailability::Unavailable => { + warn!( + sandbox_id, + "sandbox-runtime supervisor generation changed; suspending stale boundary" + ); + self.suspend_sandbox_runtime_after_dependency_failure(&lookup_api, &object) + .await; + continue; + } + SandboxRuntimeControlAvailability::Unknown => continue, + } + } + match self + .reconcile_sandbox_runtime_supervisor(&sandbox_id, namespace, desired_running) + .await + { + Ok(()) => {} + Err(KubernetesDriverError::Precondition(error)) => { + warn!(sandbox_id, %error, "sandbox-runtime companion is missing; suspending workload"); + self.suspend_sandbox_runtime_after_dependency_failure(&lookup_api, &object) + .await; + } + Err(error) => { + warn!(sandbox_id, %error, "failed to reconcile sandbox-runtime supervisor Pod"); + } + } + let availability = + sandbox_runtime_control_availability(&self.client, namespace, &sandbox_id).await; + if desired_running && availability == SandboxRuntimeControlAvailability::Unavailable { + warn!( + sandbox_id, + "sandbox-runtime supervisor is unavailable; suspending workload" + ); + self.suspend_sandbox_runtime_after_dependency_failure(&lookup_api, &object) + .await; + continue; + } + self.publish_sandbox_runtime_readiness_transition(&lookup_api, &object, availability) + .await; + } + } + + async fn reconcile_sandbox_runtime_rollback( + &self, + lookup_api: &AgentSandboxApi, + object: &DynamicObject, + sandbox_id: &str, + namespace: &str, + cr_name: &str, + ) { + let pod_name = object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(SANDBOX_POD_NAME_ANNOTATION)) + .map_or(cr_name, String::as_str); + let pod_api = Api::::namespaced(self.client.clone(), namespace); + let deadline = tokio::time::Instant::now() + KUBE_API_TIMEOUT; + match kubernetes_sandbox_pod_is_gone(&pod_api, pod_name, deadline).await { + Ok(true) => {} + Ok(false) => return, + Err(error) => { + debug!(sandbox_id, %error, "could not verify sandbox-runtime rollback; reconciliation will retry"); + return; + } + } + if let Err(error) = self + .delete_sandbox_runtime_supervisor(sandbox_id, namespace) + .await + { + warn!(sandbox_id, %error, "could not finish sandbox-runtime rollback cleanup"); + return; + } + let Some(resource_version) = object.metadata.resource_version.as_deref() else { + return; + }; + let api = + Self::agent_sandbox_api(self.client.clone(), &lookup_api.resource.version, namespace); + let patch = sandbox_runtime_rollback_completion_patch(resource_version); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + api.api + .patch(cr_name, &PatchParams::default(), &Patch::Merge(&patch)), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(error)) => { + debug!(sandbox_id, %error, "sandbox-runtime rollback completion raced; reconciliation will retry"); + } + Err(_) => { + warn!(sandbox_id, "timed out completing sandbox-runtime rollback"); + } + } + } + + async fn publish_sandbox_runtime_readiness_transition( + &self, + lookup_api: &AgentSandboxApi, + object: &DynamicObject, + availability: SandboxRuntimeControlAvailability, + ) { + let state = match availability { + SandboxRuntimeControlAvailability::Available => "ready", + SandboxRuntimeControlAvailability::Unavailable => "unavailable", + SandboxRuntimeControlAvailability::Unknown => return, + }; + if object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(ANNOTATION_SANDBOX_RUNTIME_READINESS)) + .is_some_and(|current| current == state) + { + return; + } + let (Some(name), Some(resource_version)) = ( + object.metadata.name.as_deref(), + object.metadata.resource_version.as_deref(), + ) else { + return; + }; + let namespace = object + .metadata + .namespace + .as_deref() + .unwrap_or(&self.config.namespace); + let api = + Self::agent_sandbox_api(self.client.clone(), &lookup_api.resource.version, namespace); + let patch = sandbox_runtime_readiness_transition_patch(resource_version, state); + match tokio::time::timeout( KUBE_API_TIMEOUT, - lookup_api.api.list(&lp), + api.api + .patch(name, &PatchParams::default(), &Patch::Merge(&patch)), ) .await { - Ok(Ok(list)) => { - if let Some(obj) = list.items.into_iter().next() { - match obj.metadata.name { - Some(name) => { - let ns = obj - .metadata - .namespace - .clone() - .unwrap_or_else(|| self.config.namespace.clone()); - let ws = obj - .metadata - .labels - .as_ref() - .and_then(|l| l.get(LABEL_SANDBOX_WORKSPACE).cloned()) - .unwrap_or_default(); - let pc = Preconditions { - uid: obj.metadata.uid, - resource_version: obj.metadata.resource_version, - }; - (name, ns, ws, pc) - } - None => return Ok(false), - } - } else { - debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted)"); - return Ok(false); - } - } - Ok(Err(err)) => { - warn!( - sandbox_id = %sandbox_id, - error = %err, - "Failed to list sandbox for deletion from Kubernetes" - ); - return Err(err.to_string()); + Ok(Ok(_)) => {} + Ok(Err(error)) => { + debug!(sandbox = name, %error, "sandbox-runtime readiness transition publication raced; reconciliation will retry"); } - Err(_elapsed) => { + Err(_) => { warn!( - sandbox_id = %sandbox_id, - timeout_secs = KUBE_API_TIMEOUT.as_secs(), - "Timed out listing sandbox for deletion from Kubernetes" + sandbox = name, + "timed out publishing sandbox-runtime readiness transition" ); - return Err(format!( - "timed out after {}s waiting for Kubernetes API", - KUBE_API_TIMEOUT.as_secs() - )); } - }; + } + } - let delete_api = self - .supported_agent_sandbox_api(self.client.clone(), &obj_namespace) - .await?; - let dp = DeleteParams::default().preconditions(preconditions); - match tokio::time::timeout(KUBE_API_TIMEOUT, delete_api.api.delete(&kube_name, &dp)).await { - Ok(Ok(_response)) => { - info!(sandbox_id = %sandbox_id, namespace = %obj_namespace, "Sandbox deleted from Kubernetes"); - Ok(true) + async fn suspend_sandbox_runtime_after_dependency_failure( + &self, + lookup_api: &AgentSandboxApi, + object: &DynamicObject, + ) { + let Some(name) = object.metadata.name.as_deref() else { + return; + }; + let namespace = object + .metadata + .namespace + .as_deref() + .unwrap_or(&self.config.namespace); + let api = + Self::agent_sandbox_api(self.client.clone(), &lookup_api.resource.version, namespace); + let patch = sandbox_runtime_rollback_patch( + &lookup_api.resource.version, + object + .metadata + .resource_version + .as_deref() + .unwrap_or_default(), + ); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + api.api + .patch(name, &PatchParams::default(), &Patch::Merge(&patch)), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(error)) => { + warn!(sandbox = name, %error, "failed to suspend sandbox-runtime after fence failure"); } - Ok(Err(KubeError::Api(err))) if err.code == 404 || err.code == 409 => { - debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted or replaced)"); - Ok(false) + Err(error) => { + warn!(sandbox = name, %error, "timed out suspending sandbox-runtime after fence failure"); } - Ok(Err(err)) => { - warn!( - sandbox_id = %sandbox_id, - error = %err, - "Failed to delete sandbox from Kubernetes" - ); - Err(err.to_string()) + } + } + + async fn reap_stale_sandbox_runtime_bootstrap( + &self, + lookup_api: &AgentSandboxApi, + object: &DynamicObject, + ) { + if !sandbox_runtime_bootstrap_is_stale( + object, + SystemTime::now(), + SANDBOX_RUNTIME_BOOTSTRAP_GRACE, + ) { + return; + } + if sandbox_runtime_bootstrap_operation(object) != Some("create") { + warn!( + sandbox = object.metadata.name.as_deref().unwrap_or(""), + "suspending stale sandbox-runtime restart bootstrap" + ); + self.suspend_sandbox_runtime_after_dependency_failure(lookup_api, object) + .await; + return; + } + let (Some(name), Some(uid), Some(resource_version)) = ( + object.metadata.name.as_deref(), + object.metadata.uid.clone(), + object.metadata.resource_version.clone(), + ) else { + return; + }; + let namespace = object + .metadata + .namespace + .as_deref() + .unwrap_or(&self.config.namespace); + let api = + Self::agent_sandbox_api(self.client.clone(), &lookup_api.resource.version, namespace); + let params = DeleteParams::default().preconditions(Preconditions { + uid: Some(uid), + resource_version: Some(resource_version), + }); + match tokio::time::timeout(KUBE_API_TIMEOUT, api.api.delete(name, ¶ms)).await { + Ok(Ok(_)) => warn!( + sandbox = name, + "rolled back stale fail-closed sandbox-runtime bootstrap" + ), + Ok(Err(KubeError::Api(error))) if error.code == 404 || error.code == 409 => {} + Ok(Err(error)) => { + warn!(sandbox = name, %error, "failed to roll back stale sandbox-runtime bootstrap"); } - Err(_elapsed) => { + Err(_) => { warn!( - sandbox_id = %sandbox_id, - timeout_secs = KUBE_API_TIMEOUT.as_secs(), - "Timed out deleting sandbox from Kubernetes" + sandbox = name, + "timed out rolling back stale sandbox-runtime bootstrap" ); - Err(format!( - "timed out after {}s waiting for Kubernetes API", - KUBE_API_TIMEOUT.as_secs() - )) } } } - pub async fn sandbox_exists(&self, sandbox_id: &str) -> Result { - let agent_sandbox_api = self - .supported_sandbox_api_for_lookup(self.client.clone()) - .await?; - let selector = self.sandbox_lookup_selector(sandbox_id); - let lp = ListParams::default().labels(&selector); - match tokio::time::timeout(KUBE_API_TIMEOUT, agent_sandbox_api.api.list(&lp)).await { - Ok(Ok(list)) => Ok(!list.items.is_empty()), - Ok(Err(err)) => Err(err.to_string()), - Err(_elapsed) => Err(format!( - "timed out after {}s waiting for Kubernetes API", - KUBE_API_TIMEOUT.as_secs() - )), + async fn complete_sandbox_runtime_bootstrap( + &self, + lookup_api: &AgentSandboxApi, + object: &DynamicObject, + ) { + let (Some(name), Some(resource_version)) = ( + object.metadata.name.as_deref(), + object.metadata.resource_version.as_deref(), + ) else { + return; + }; + let namespace = object + .metadata + .namespace + .as_deref() + .unwrap_or(&self.config.namespace); + let api = + Self::agent_sandbox_api(self.client.clone(), &lookup_api.resource.version, namespace); + let patch = sandbox_runtime_bootstrap_completion_patch(resource_version); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + api.api + .patch(name, &PatchParams::default(), &Patch::Merge(&patch)), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(error)) => { + debug!(sandbox = name, %error, "sandbox-runtime bootstrap completion raced; reconciliation will retry"); + } + Err(_) => warn!( + sandbox = name, + "timed out completing sandbox-runtime bootstrap" + ), + } + } + + async fn reconcile_sandbox_runtime_supervisor( + &self, + sandbox_id: &str, + namespace: &str, + desired_running: bool, + ) -> Result<(), KubernetesDriverError> { + if !desired_running { + return self + .delete_sandbox_runtime_supervisor(sandbox_id, namespace) + .await; + } + let names = SandboxRuntimeNames::new(sandbox_id); + let services = Api::::namespaced(self.client.clone(), namespace); + let service_exists = + tokio::time::timeout(KUBE_API_TIMEOUT, services.get_opt(&names.boundary_service)) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out reading sandbox-runtime boundary Service".to_string(), + ) + })? + .map_err(KubernetesDriverError::from_kube)? + .is_some(); + if !service_exists { + return Err(KubernetesDriverError::Precondition(format!( + "sandbox-runtime boundary Service {} is missing and its allocated address cannot be safely reconstructed", + names.boundary_service + ))); } + let pods = Api::::namespaced(self.client.clone(), namespace); + tokio::time::timeout( + KUBE_API_TIMEOUT, + pods.get_opt(&names.supervisor_pod), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out reading sandbox-runtime supervisor Pod".to_string(), + ) + })? + .map_err(KubernetesDriverError::from_kube)? + .ok_or_else(|| { + KubernetesDriverError::Precondition(format!( + "sandbox-runtime supervisor Pod {} is missing; start a fresh sandbox generation", + names.supervisor_pod + )) + })?; + Ok(()) + } + + fn spawn_sandbox_runtime_periodic_reconcile( + &self, + tx: mpsc::Sender>, + ) { + let driver = self.clone(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(SANDBOX_RUNTIME_RECONCILE_INTERVAL); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + interval.tick().await; + loop { + tokio::select! { + _ = interval.tick() => { + driver.reconcile_sandbox_runtime_resources().await; + if let Ok(sandboxes) = driver.list_sandboxes().await { + for sandbox in sandboxes { + if tx.send(Ok(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { sandbox: Some(sandbox) }, + )), + })).await.is_err() { + return; + } + } + } + } + () = tx.closed() => return, + } + } + }); } // Kept `async` to match the gRPC handler signature in `grpc.rs`, which awaits this method. #[allow(clippy::unused_async)] pub async fn watch_sandboxes(&self) -> Result { + self.reconcile_sandbox_runtime_resources().await; if self.config.is_multi_namespace() { self.watch_sandboxes_cluster_wide().await } else { @@ -1957,6 +3890,8 @@ impl KubernetesComputeDriver { ) .boxed(); let (tx, rx) = mpsc::channel(256); + self.spawn_sandbox_runtime_periodic_reconcile(tx.clone()); + let readiness_client = self.watch_client.clone(); tokio::spawn(async move { let mut sandbox_name_to_id = std::collections::HashMap::::new(); @@ -1966,7 +3901,7 @@ impl KubernetesComputeDriver { tokio::select! { event = sandbox_stream.next() => match event { Some(Event::Apply(obj) | Event::InitApply(obj)) => { - if let Ok((kube_name, sandbox)) = sandbox_from_object(&namespace, obj) { + if let Ok((kube_name, sandbox)) = sandbox_from_object_with_sandbox_runtime_readiness(&readiness_client, &namespace, obj).await { update_indexes(&mut sandbox_name_to_id, &mut agent_pod_to_id, &kube_name, &sandbox); let event = WatchSandboxesEvent { payload: Some(watch_sandboxes_event::Payload::Sandbox( @@ -2002,7 +3937,7 @@ impl KubernetesComputeDriver { } }, event = event_stream.next() => match event { - Some(Event::Apply(obj)) => { + Some(Event::Apply(obj) | Event::InitApply(obj)) => { if let Some((sandbox_id, event)) = map_kube_event_to_platform( &sandbox_name_to_id, &agent_pod_to_id, @@ -2018,8 +3953,8 @@ impl KubernetesComputeDriver { } } } - Some(Event::Delete(_) | Event::InitApply(_) | Event::InitDone) => {} - Some(Event::Init) => { + Some(Event::Delete(_)) => {} + Some(Event::Init | Event::InitDone) => { debug!(namespace = %namespace, "Kubernetes event watcher restarted"); } None => { @@ -2051,18 +3986,225 @@ impl KubernetesComputeDriver { ) .boxed(); - Ok(cluster_wide_watch_stream( - sandbox_stream, - self.config.namespace.clone(), - )) - } + Ok(cluster_wide_watch_stream( + sandbox_stream, + self.config.namespace.clone(), + self.watch_client.clone(), + self.clone(), + )) + } +} + +fn sandbox_runtime_bootstrap_completion_patch(resource_version: &str) -> serde_json::Value { + serde_json::json!({ + "metadata": { + "resourceVersion": resource_version, + "annotations": { + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_SESSION: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_READINESS: "ready", + } + } + }) +} + +fn spawn_sandbox_runtime_bootstrap_completion( + pods: Api, + sandboxes: Api, + supervisor_pod_name: String, + sandbox_name: String, + expected_sandbox_uid: Option, +) { + tokio::spawn(async move { + let deadline = tokio::time::Instant::now() + SANDBOX_RUNTIME_BOOTSTRAP_GRACE; + let available = |pod: Option<&Pod>| { + pod.is_some_and(|pod| { + sandbox_runtime_control_availability_from_pod(pod) + == SandboxRuntimeControlAvailability::Available + }) + }; + match tokio::time::timeout_at( + deadline, + await_condition(pods, &supervisor_pod_name, available), + ) + .await + { + Ok(Ok(Some(_))) => {} + Ok(Ok(None)) | Err(_) => return, + Ok(Err(error)) => { + debug!(%error, sandbox = sandbox_name, "sandbox-runtime bootstrap availability watch failed; reconciliation will retry"); + return; + } + } + + // A replacement control can become Available before the Agent + // Sandbox controller has replaced the prior Suspended condition. + // Clearing the bootstrap marker in that window publishes a terminal + // Stopped event to the gateway. Wait for the controller's real Ready + // condition so the marker removal and readiness annotation expose one + // causally complete transition for both API versions. + let runtime_ready = |object: Option<&DynamicObject>| { + object.is_some_and(|object| { + object.metadata.uid == expected_sandbox_uid + && sandbox_runtime_bootstrap_in_progress(object) + && sandbox_runtime_runtime_is_ready(object) + }) + }; + let object = match tokio::time::timeout_at( + deadline, + await_condition(sandboxes.clone(), &sandbox_name, runtime_ready), + ) + .await + { + Ok(Ok(Some(object))) => object, + Ok(Ok(None)) | Err(_) => return, + Ok(Err(error)) => { + debug!(%error, sandbox = sandbox_name, "sandbox-runtime runtime readiness watch failed; reconciliation will retry"); + return; + } + }; + let Some(resource_version) = object.metadata.resource_version.as_deref() else { + return; + }; + let patch = sandbox_runtime_bootstrap_completion_patch(resource_version); + if let Err(error) = sandboxes + .patch( + &sandbox_name, + &PatchParams::default(), + &Patch::Merge(&patch), + ) + .await + { + debug!(%error, sandbox = sandbox_name, "sandbox-runtime bootstrap completion raced; reconciliation will retry"); + } + }); +} + +fn sandbox_runtime_runtime_is_ready(object: &DynamicObject) -> bool { + let Some(generation) = object.metadata.generation else { + return false; + }; + let Some(conditions) = object + .data + .pointer("/status/conditions") + .and_then(serde_json::Value::as_array) + else { + return false; + }; + let observes_current_generation = |condition: &serde_json::Value| { + condition + .get("observedGeneration") + .and_then(serde_json::Value::as_i64) + == Some(generation) + }; + let ready = conditions.iter().any(|condition| { + condition.get("type").and_then(serde_json::Value::as_str) == Some("Ready") + && condition.get("status").and_then(serde_json::Value::as_str) == Some("True") + && observes_current_generation(condition) + }); + let suspended = conditions.iter().any(|condition| { + condition.get("type").and_then(serde_json::Value::as_str) + == Some(SANDBOX_SUSPENDED_CONDITION) + && condition.get("status").and_then(serde_json::Value::as_str) == Some("True") + && observes_current_generation(condition) + }); + ready && !suspended +} + +fn sandbox_runtime_bootstrap_in_progress(object: &DynamicObject) -> bool { + object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING)) + .is_some_and(|value| value == "true") +} + +fn sandbox_runtime_bootstrap_operation(object: &DynamicObject) -> Option<&str> { + object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION)) + .map(String::as_str) +} + +fn sandbox_runtime_bootstrap_phase(object: &DynamicObject) -> Option { + object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE)) + .and_then(|value| SandboxRuntimeBootstrapPhase::parse(value)) +} + +fn sandbox_runtime_generation(object: &DynamicObject) -> Option<&str> { + object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(ANNOTATION_SANDBOX_RUNTIME_GENERATION)) + .map(String::as_str) +} + +fn sandbox_runtime_bootstrap_phase_patch( + resource_version: &str, + phase: SandboxRuntimeBootstrapPhase, +) -> serde_json::Value { + serde_json::json!({ + "metadata": { + "resourceVersion": resource_version, + "annotations": { + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE: phase.as_str(), + } + } + }) +} + +fn sandbox_runtime_bootstrap_is_stale( + object: &DynamicObject, + now: SystemTime, + minimum_age: Duration, +) -> bool { + let Some(started_at_ms) = object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT)) + .and_then(|value| value.parse::().ok()) + else { + return true; + }; + now.duration_since(SystemTime::UNIX_EPOCH + Duration::from_millis(started_at_ms)) + .is_ok_and(|age| age >= minimum_age) +} + +fn sandbox_runtime_readiness_transition_patch( + resource_version: &str, + state: &str, +) -> serde_json::Value { + serde_json::json!({ + "metadata": { + "resourceVersion": resource_version, + "annotations": { ANNOTATION_SANDBOX_RUNTIME_READINESS: state }, + } + }) } -fn cluster_wide_watch_stream(mut sandbox_stream: S, default_namespace: String) -> WatchStream +fn cluster_wide_watch_stream( + mut sandbox_stream: S, + default_namespace: String, + readiness_client: Client, + driver: KubernetesComputeDriver, +) -> WatchStream where S: Stream> + Send + Unpin + 'static, { let (tx, rx) = mpsc::channel(256); + driver.spawn_sandbox_runtime_periodic_reconcile(tx.clone()); tokio::spawn(async move { loop { @@ -2071,7 +4213,7 @@ where Some(Event::Apply(obj) | Event::InitApply(obj)) => { let ns = obj.metadata.namespace.clone() .unwrap_or_else(|| default_namespace.clone()); - if let Ok((_kube_name, sandbox)) = sandbox_from_object(&ns, obj) { + if let Ok((_kube_name, sandbox)) = sandbox_from_object_with_sandbox_runtime_readiness(&readiness_client, &ns, obj).await { let event = WatchSandboxesEvent { payload: Some(watch_sandboxes_event::Payload::Sandbox( WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } @@ -2422,8 +4564,53 @@ fn validate_pod_uid(pod: &Pod, expected_uid: &str) -> Result<(), tonic::Status> )) } +fn require_proxy_control_authentication(via_proxy_control: bool) -> Result<(), tonic::Status> { + if via_proxy_control { + Ok(()) + } else { + Err(tonic::Status::permission_denied( + "sandbox JWT authentication must originate from the paired supervisor Pod", + )) + } +} + +#[allow(clippy::result_large_err)] +fn validate_proxy_control_labels(pod: &Pod, sandbox_id: &str) -> Result<(), tonic::Status> { + validate_proxy_control_labels_from_metadata(&pod.metadata, sandbox_id) +} + +#[allow(clippy::result_large_err)] +fn validate_proxy_control_labels_from_metadata( + metadata: &ObjectMeta, + sandbox_id: &str, +) -> Result<(), tonic::Status> { + let labels = metadata.labels.as_ref().ok_or_else(|| { + tonic::Status::permission_denied("control workload has no sandbox-runtime labels") + })?; + let expected_pair = crate::sandbox_runtime::pair_label_value(sandbox_id); + let matches = labels + .get(BOUNDARY_ROLE_LABEL) + .is_some_and(|role| role == "supervisor") + && labels + .get(BOUNDARY_PAIR_LABEL) + .is_some_and(|pair| pair == &expected_pair) + && labels + .get(LABEL_SANDBOX_ID) + .is_some_and(|actual| actual == sandbox_id); + if matches { + Ok(()) + } else { + Err(tonic::Status::permission_denied( + "control workload sandbox-runtime labels do not match the sandbox identity", + )) + } +} + #[allow(clippy::result_large_err)] -fn sandbox_owner_reference(pod: &Pod) -> Result<&OwnerReference, tonic::Status> { +fn sandbox_owner_reference( + pod: &Pod, + require_controller: bool, +) -> Result<&OwnerReference, tonic::Status> { let mut owners = pod .metadata .owner_references @@ -2439,9 +4626,10 @@ fn sandbox_owner_reference(pod: &Pod) -> Result<&OwnerReference, tonic::Status> }); let owner = owners .next() - .ok_or_else(|| tonic::Status::permission_denied("pod is not controlled by a Sandbox"))?; + .ok_or_else(|| tonic::Status::permission_denied("pod is not owned by a Sandbox"))?; if owners.next().is_some() - || owner.controller != Some(true) + || (require_controller && owner.controller != Some(true)) + || (!require_controller && owner.controller == Some(true)) || owner.name.is_empty() || owner.uid.is_empty() { @@ -2548,6 +4736,304 @@ fn sandbox_from_object(namespace: &str, obj: DynamicObject) -> Result<(String, S )) } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum SandboxRuntimeControlAvailability { + Available, + Unavailable, + Unknown, +} + +fn sandbox_runtime_control_availability_from_pod(pod: &Pod) -> SandboxRuntimeControlAvailability { + if pod.metadata.deletion_timestamp.is_none() + && pod + .status + .as_ref() + .and_then(|status| status.conditions.as_ref()) + .is_some_and(|conditions| { + conditions + .iter() + .any(|condition| condition.type_ == "Ready" && condition.status == "True") + }) + { + SandboxRuntimeControlAvailability::Available + } else { + SandboxRuntimeControlAvailability::Unavailable + } +} + +async fn sandbox_runtime_control_availability( + client: &Client, + namespace: &str, + sandbox_id: &str, +) -> SandboxRuntimeControlAvailability { + let names = SandboxRuntimeNames::new(sandbox_id); + let pods = Api::::namespaced(client.clone(), namespace); + let services = Api::::namespaced(client.clone(), namespace); + let policies = Api::::namespaced(client.clone(), namespace); + let supervisor = Box::pin(tokio::time::timeout( + KUBE_API_TIMEOUT, + pods.get_opt(&names.supervisor_pod), + )); + let service = Box::pin(tokio::time::timeout( + KUBE_API_TIMEOUT, + services.get_opt(&names.boundary_service), + )); + let fence = Box::pin(tokio::time::timeout( + KUBE_API_TIMEOUT, + policies.get_opt(&names.workload_policy), + )); + let supervisor_fence = Box::pin(tokio::time::timeout( + KUBE_API_TIMEOUT, + policies.get_opt(&names.supervisor_policy), + )); + let (supervisor, service, fence, supervisor_fence) = + tokio::join!(supervisor, service, fence, supervisor_fence); + let control = match supervisor { + Ok(Ok(Some(pod))) => sandbox_runtime_control_availability_from_pod(&pod), + Ok(Ok(None)) => SandboxRuntimeControlAvailability::Unavailable, + Ok(Err(error)) => { + warn!(sandbox_id, %error, "could not determine sandbox-runtime control availability"); + SandboxRuntimeControlAvailability::Unknown + } + Err(_) => { + warn!( + sandbox_id, + "timed out checking sandbox-runtime control availability" + ); + SandboxRuntimeControlAvailability::Unknown + } + }; + let service = match service { + Ok(Ok(Some(_))) => SandboxRuntimeControlAvailability::Available, + Ok(Ok(None)) => SandboxRuntimeControlAvailability::Unavailable, + Ok(Err(error)) => { + warn!(sandbox_id, %error, "could not determine sandbox-runtime boundary Service availability"); + SandboxRuntimeControlAvailability::Unknown + } + Err(_) => { + warn!( + sandbox_id, + "timed out checking sandbox-runtime boundary Service availability" + ); + SandboxRuntimeControlAvailability::Unknown + } + }; + + // Supervisor readiness alone is not sufficient: deletion of the shared + // fence would otherwise leave a live boundary with direct pod + // egress while the driver continued to publish Ready. + let fence = match fence { + Ok(Ok(Some(_))) => SandboxRuntimeControlAvailability::Available, + Ok(Ok(None)) => SandboxRuntimeControlAvailability::Unavailable, + Ok(Err(error)) => { + warn!(sandbox_id, %error, "could not determine sandbox-runtime workload fence availability"); + SandboxRuntimeControlAvailability::Unknown + } + Err(_) => { + warn!( + sandbox_id, + "timed out checking sandbox-runtime workload fence availability" + ); + SandboxRuntimeControlAvailability::Unknown + } + }; + let supervisor_fence = match supervisor_fence { + Ok(Ok(Some(_))) => SandboxRuntimeControlAvailability::Available, + Ok(Ok(None)) => SandboxRuntimeControlAvailability::Unavailable, + Ok(Err(error)) => { + warn!(sandbox_id, %error, "could not determine sandbox-runtime supervisor fence availability"); + SandboxRuntimeControlAvailability::Unknown + } + Err(_) => { + warn!( + sandbox_id, + "timed out checking sandbox-runtime supervisor fence availability" + ); + SandboxRuntimeControlAvailability::Unknown + } + }; + let dependencies = [control, service, fence, supervisor_fence]; + if dependencies.contains(&SandboxRuntimeControlAvailability::Unavailable) { + SandboxRuntimeControlAvailability::Unavailable + } else if dependencies.contains(&SandboxRuntimeControlAvailability::Unknown) { + SandboxRuntimeControlAvailability::Unknown + } else { + SandboxRuntimeControlAvailability::Available + } +} + +async fn sandbox_runtime_workload_generation_matches( + client: &Client, + namespace: &str, + pod_name: &str, + sandbox: &DynamicObject, +) -> SandboxRuntimeControlAvailability { + let Some(expected_uid) = sandbox + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID)) + else { + return SandboxRuntimeControlAvailability::Unavailable; + }; + let pods = Api::::namespaced(client.clone(), namespace); + match tokio::time::timeout(KUBE_API_TIMEOUT, pods.get_opt(pod_name)).await { + Ok(Ok(Some(pod))) if pod.metadata.uid.as_deref() == Some(expected_uid.as_str()) => { + SandboxRuntimeControlAvailability::Available + } + Ok(Ok(_)) => SandboxRuntimeControlAvailability::Unavailable, + Ok(Err(error)) => { + warn!(pod = pod_name, %error, "could not verify sandbox-runtime workload generation"); + SandboxRuntimeControlAvailability::Unknown + } + Err(_) => { + warn!( + pod = pod_name, + "timed out checking sandbox-runtime workload generation" + ); + SandboxRuntimeControlAvailability::Unknown + } + } +} + +async fn sandbox_runtime_supervisor_generation_matches( + client: &Client, + namespace: &str, + sandbox_id: &str, + sandbox: &DynamicObject, +) -> SandboxRuntimeControlAvailability { + let Some(expected_uid) = sandbox + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID)) + else { + return SandboxRuntimeControlAvailability::Unavailable; + }; + let names = SandboxRuntimeNames::new(sandbox_id); + let pods = Api::::namespaced(client.clone(), namespace); + match tokio::time::timeout(KUBE_API_TIMEOUT, pods.get_opt(&names.supervisor_pod)).await { + Ok(Ok(Some(pod))) if pod.metadata.uid.as_deref() == Some(expected_uid.as_str()) => { + SandboxRuntimeControlAvailability::Available + } + Ok(Ok(_)) => SandboxRuntimeControlAvailability::Unavailable, + Ok(Err(error)) => { + warn!(pod = names.supervisor_pod, %error, "could not verify sandbox-runtime supervisor generation"); + SandboxRuntimeControlAvailability::Unknown + } + Err(_) => { + warn!( + pod = names.supervisor_pod, + "timed out checking sandbox-runtime supervisor generation" + ); + SandboxRuntimeControlAvailability::Unknown + } + } +} + +async fn sandbox_from_object_with_sandbox_runtime_readiness( + client: &Client, + namespace: &str, + obj: DynamicObject, +) -> Result<(String, Sandbox), String> { + let bootstrapping = sandbox_runtime_bootstrap_in_progress(&obj); + let sandbox_id = sandbox_id_from_object(&obj).unwrap_or_default(); + let object_namespace = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| namespace.to_string()); + let (name, mut sandbox) = sandbox_from_object(namespace, obj.clone())?; + if bootstrapping { + mark_sandbox_runtime_bootstrapping(&mut sandbox); + } + if !sandbox_id.is_empty() { + let dependencies = Box::pin(sandbox_runtime_control_availability( + client, + &object_namespace, + &sandbox_id, + )); + let workload_generation = Box::pin(sandbox_runtime_workload_generation_matches( + client, + &object_namespace, + &name, + &obj, + )); + let supervisor_generation = Box::pin(sandbox_runtime_supervisor_generation_matches( + client, + &object_namespace, + &sandbox_id, + &obj, + )); + let (dependencies, workload_generation, supervisor_generation) = + tokio::join!(dependencies, workload_generation, supervisor_generation); + if dependencies != SandboxRuntimeControlAvailability::Available + || workload_generation != SandboxRuntimeControlAvailability::Available + || supervisor_generation != SandboxRuntimeControlAvailability::Available + { + mark_sandbox_runtime_control_unavailable(&mut sandbox); + } + } + Ok((name, sandbox)) +} + +fn mark_sandbox_runtime_bootstrapping(sandbox: &mut Sandbox) { + if let Some(status) = sandbox.status.as_mut() { + status.conditions.retain(|condition| { + condition.r#type != SANDBOX_SUSPENDED_CONDITION && condition.r#type != "Bootstrapping" + }); + status.conditions.push(SandboxCondition { + r#type: "Bootstrapping".to_string(), + status: "True".to_string(), + reason: "SandboxRuntimeGenerationStarting".to_string(), + message: "replacement sandbox-runtime generation is starting".to_string(), + last_transition_time: String::new(), + }); + } + mark_sandbox_runtime_control_unavailable(sandbox); +} + +fn mark_sandbox_runtime_control_unavailable(sandbox: &mut Sandbox) { + const REASON: &str = "DependenciesNotReady"; + const MESSAGE: &str = "sandbox-runtime enforcement dependencies are not ready"; + let Some(status) = sandbox.status.as_mut() else { + return; + }; + if let Some(ready) = status + .conditions + .iter_mut() + .find(|condition| condition.r#type == "Ready") + { + ready.status = "False".to_string(); + ready.reason = REASON.to_string(); + ready.message = MESSAGE.to_string(); + } else { + status.conditions.push(SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: REASON.to_string(), + message: MESSAGE.to_string(), + last_transition_time: String::new(), + }); + } +} + +fn sandbox_runtime_should_run(obj: &DynamicObject) -> bool { + if let Some(mode) = obj + .data + .get("spec") + .and_then(|spec| spec.get("operatingMode")) + .and_then(serde_json::Value::as_str) + { + return !mode.eq_ignore_ascii_case("Suspended"); + } + obj.data + .get("spec") + .and_then(|spec| spec.get("replicas")) + .and_then(serde_json::Value::as_i64) + .is_none_or(|replicas| replicas > 0) +} + fn update_indexes( sandbox_name_to_id: &mut std::collections::HashMap, agent_pod_to_id: &mut std::collections::HashMap, @@ -2642,740 +5128,276 @@ fn attach_kube_progress_metadata( "Pulling" => { mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); if let Some(image) = pulling_image_from_kube_message(message) { - mark_progress_detail(metadata, image); - } - } - "Pulled" => { - let label = pulled_image_label(message); - mark_progress_complete(metadata, PROGRESS_STEP_PULLING_IMAGE, label); - mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); - } - _ => {} - } -} - -fn pulling_image_from_kube_message(message: &str) -> Option { - let image = message - .strip_prefix("Pulling image ") - .map(str::trim) - .map(|value| value.trim_matches('"'))?; - (!image.is_empty()).then(|| image.to_string()) -} - -fn pulled_image_label(message: &str) -> String { - extract_image_size(message).map_or_else( - || "Image pulled".to_string(), - |bytes| format!("Image pulled ({})", format_bytes(bytes)), - ) -} - -fn extract_image_size(message: &str) -> Option { - let size_prefix = "Image size: "; - let start = message.find(size_prefix)? + size_prefix.len(); - let rest = &message[start..]; - let end = rest.find(' ')?; - rest[..end].parse().ok() -} - -/// Path where the supervisor binary is mounted inside the agent container. -const SUPERVISOR_MOUNT_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_DIR; - -/// Name of the volume used to side-load the supervisor binary. -const SUPERVISOR_VOLUME_NAME: &str = "openshell-supervisor-bin"; - -/// Name of the init container that installs the supervisor binary. -const SUPERVISOR_INIT_CONTAINER_NAME: &str = "openshell-supervisor-install"; - -/// Name of the init container that prepares pod-level sidecar networking. -const SUPERVISOR_NETWORK_INIT_CONTAINER_NAME: &str = "openshell-network-init"; - -/// Container name for the network-only supervisor sidecar. -const SUPERVISOR_NETWORK_SIDECAR_NAME: &str = "openshell-supervisor-network"; - -/// UID used by strict process/binary-aware sidecars so Kubernetes grants the -/// requested capability set into the effective set without privilege escalation. -const BINARY_AWARE_SIDECAR_PROXY_UID: u32 = 0; - -/// Shared volume used by the network sidecar and process-only supervisor for -/// local coordination in sidecar topology. -const SIDECAR_STATE_VOLUME_NAME: &str = "openshell-sidecar-state"; -const SIDECAR_STATE_MOUNT_PATH: &str = openshell_core::container_paths::SIDECAR_RUN_ROOT; -const SIDECAR_CONTROL_SOCKET: &str = openshell_core::container_paths::SIDECAR_CONTROL_SOCKET; -// Linux abstract socket names are scoped to the pod's shared network namespace. -// Unlike a filesystem socket in the shared state volume, the workload cannot -// unlink and replace this relay endpoint after the trusted supervisor binds it. -const SIDECAR_SSH_SOCKET_FILE: &str = "@openshell-sidecar-ssh"; - -/// Shared TLS work directory. The network sidecar writes the proxy CA bundle -/// here, while the agent container consumes it after sidecar bootstrap. -const SIDECAR_TLS_VOLUME_NAME: &str = "openshell-supervisor-tls"; -const SIDECAR_TLS_MOUNT_PATH: &str = openshell_core::container_paths::SIDECAR_TLS_DIR; -const SIDECAR_CLIENT_TLS_MOUNT_PATH: &str = openshell_core::container_paths::SIDECAR_CLIENT_TLS_DIR; - -/// Build the emptyDir volume that holds the supervisor binary. -/// -/// The init container writes the binary here; the agent container reads it. -fn supervisor_volume() -> serde_json::Value { - serde_json::json!({ - "name": SUPERVISOR_VOLUME_NAME, - "emptyDir": {} - }) -} - -/// Build the read-only volume mount for the supervisor binary in the agent container. -fn supervisor_volume_mount() -> serde_json::Value { - serde_json::json!({ - "name": SUPERVISOR_VOLUME_NAME, - "mountPath": SUPERVISOR_MOUNT_PATH, - "readOnly": true - }) -} - -/// Build an image volume that mounts the supervisor OCI image directly. -/// -/// Requires Kubernetes >= v1.33 (`ImageVolume` beta) or >= v1.36 (GA). -/// The entire image filesystem is mounted read-only, making the binary -/// available at `{SUPERVISOR_MOUNT_PATH}/openshell-sandbox`. -fn supervisor_image_volume( - supervisor_image: &str, - supervisor_image_pull_policy: Option<&str>, -) -> serde_json::Value { - let mut image_spec = serde_json::json!({ - "reference": supervisor_image, - }); - if let Some(policy) = supervisor_image_pull_policy { - image_spec["pullPolicy"] = serde_json::json!(policy); - } - serde_json::json!({ - "name": SUPERVISOR_VOLUME_NAME, - "image": image_spec - }) -} - -/// Build the init container that copies the supervisor binary into the emptyDir. -/// -/// The supervisor image contains the supervisor binary at `/openshell-sandbox`. -/// We invoke that binary with the `copy-self` subcommand so it copies itself -/// into the shared emptyDir volume, where the agent container then executes it -/// from a fixed, writable path. This pattern (binary self-copy) avoids requiring -/// `sh`/`cp` in the supervisor image and mirrors the approach used by argoexec's -/// emissary executor. -fn supervisor_init_container( - supervisor_image: &str, - supervisor_image_pull_policy: Option<&str>, -) -> serde_json::Value { - let installed_path = format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"); - let mut spec = serde_json::json!({ - "name": SUPERVISOR_INIT_CONTAINER_NAME, - "image": supervisor_image, - "command": [ - SUPERVISOR_IMAGE_BINARY_PATH, - "copy-self", - installed_path, - ], - "securityContext": {"runAsUser": 0}, - "volumeMounts": [{ - "name": SUPERVISOR_VOLUME_NAME, - "mountPath": SUPERVISOR_MOUNT_PATH, - "readOnly": false - }] - }); - if let Some(policy) = supervisor_image_pull_policy { - spec["imagePullPolicy"] = serde_json::json!(policy); - } - spec -} - -fn apply_supervisor_binary_source( - spec: &mut serde_json::Map, - supervisor_image: &str, - supervisor_image_pull_policy: Option<&str>, - method: SupervisorSideloadMethod, -) { - let volumes = spec - .entry("volumes") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(volumes) = volumes { - match method { - SupervisorSideloadMethod::ImageVolume => { - volumes.push(supervisor_image_volume( - supervisor_image, - supervisor_image_pull_policy, - )); - } - SupervisorSideloadMethod::InitContainer => { - volumes.push(supervisor_volume()); - } - } - } - - if method == SupervisorSideloadMethod::InitContainer { - let init_containers = spec - .entry("initContainers") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(init_containers) = init_containers { - init_containers.push(supervisor_init_container( - supervisor_image, - supervisor_image_pull_policy, - )); - } - } -} - -/// Apply supervisor side-load transforms to an already-built pod template JSON. -/// -/// Depending on the sideload method: -/// - **`ImageVolume`**: mounts the supervisor OCI image directly as a read-only -/// volume (no init container needed, requires K8s >= v1.33). -/// - **`InitContainer`**: injects an emptyDir volume and an init container that -/// copies the supervisor binary from the supervisor image into that volume. -/// -/// In both cases, the agent container gets a command override to run the -/// side-loaded binary as root so it can create network namespaces, set up the -/// proxy, and configure Landlock/seccomp. -#[allow(clippy::similar_names)] -fn apply_supervisor_sideload_with_params( - pod_template: &mut serde_json::Value, - params: &SandboxPodParams<'_>, -) { - let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { - return; - }; - - apply_supervisor_binary_source( - spec, - params.supervisor_image, - params.supervisor_image_pull_policy, - params.supervisor_sideload_method, - ); - - // Find the agent container and add volume mount + command override - let Some(containers) = spec.get_mut("containers").and_then(|v| v.as_array_mut()) else { - return; - }; - - let mut target_index = None; - for (i, c) in containers.iter().enumerate() { - if c.get("name").and_then(|v| v.as_str()) == Some("agent") { - target_index = Some(i); - break; - } - } - let index = target_index.unwrap_or(0); - - if let Some(container) = containers.get_mut(index).and_then(|v| v.as_object_mut()) { - // Override command to use the side-loaded supervisor binary - let mut command = vec![ - format!("{}/openshell-sandbox", SUPERVISOR_MOUNT_PATH), - "--workdir".to_string(), - driver_mounts::DEFAULT_WORKSPACE_ROOT.to_string(), - ]; - command.extend(upstream_proxy_cli_args(params)); - container.insert("command".to_string(), serde_json::json!(command)); - - // Force the supervisor to run as root (UID 0). Sandbox images may set - // a non-root USER directive (e.g. `USER sandbox`), but the supervisor - // needs root to create network namespaces, set up the proxy, and - // configure Landlock/seccomp. The supervisor itself drops privileges - // for child processes via the policy's `run_as_user`/`run_as_group`. - let security_context = container - .entry("securityContext") - .or_insert_with(|| serde_json::json!({})); - if let Some(sc) = security_context.as_object_mut() { - sc.insert("runAsUser".to_string(), serde_json::json!(0)); - } - - // Add volume mount - let volume_mounts = container - .entry("volumeMounts") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(volume_mounts) = volume_mounts { - volume_mounts.push(supervisor_volume_mount()); - } - - // Inject the protected resolved identity contract. Clearing the OCI - // input prevents image or user environment from selecting a - // conflicting identity path. - let env = container - .entry("env") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(env) = env { - apply_resolved_identity_env(env, params.sandbox_uid, params.sandbox_gid); - } - if has_upstream_proxy_credentials(params) { - let volume_mounts = container - .entry("volumeMounts") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(volume_mounts) = volume_mounts { - volume_mounts.push(upstream_proxy_auth_volume_mount()); + mark_progress_detail(metadata, image); } } + "Pulled" => { + let label = pulled_image_label(message); + mark_progress_complete(metadata, PROGRESS_STEP_PULLING_IMAGE, label); + mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); + } + _ => {} } } -#[cfg(test)] -#[allow(clippy::similar_names)] -fn apply_supervisor_sideload( - pod_template: &mut serde_json::Value, - supervisor_image: &str, - supervisor_image_pull_policy: Option<&str>, - method: SupervisorSideloadMethod, - sandbox_uid: u32, - sandbox_gid: u32, -) { - let params = SandboxPodParams { - supervisor_image, - supervisor_image_pull_policy, - supervisor_sideload_method: method, - sandbox_uid, - sandbox_gid, - ..SandboxPodParams::default() - }; - apply_supervisor_sideload_with_params(pod_template, ¶ms); -} - -fn upstream_proxy_cli_args(params: &SandboxPodParams<'_>) -> Vec { - let mut args = Vec::new(); - if let Some(url) = params.https_proxy { - args.extend(["--upstream-proxy".to_string(), url.to_string()]); - } - if let Some(list) = params.no_proxy { - args.extend(["--upstream-no-proxy".to_string(), list.to_string()]); - } - if has_upstream_proxy_credentials(params) { - args.extend([ - "--upstream-proxy-auth-file".to_string(), - openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH.to_string(), - ]); - } - if params.proxy_auth_allow_insecure { - args.push("--upstream-proxy-auth-allow-insecure".to_string()); - } - if params.proxy_connect_by_hostname { - args.push("--upstream-proxy-connect-by-hostname".to_string()); - } - args -} - -fn upstream_proxy_auth_volume_mount() -> serde_json::Value { - serde_json::json!({ - "name": UPSTREAM_PROXY_AUTH_VOLUME_NAME, - "mountPath": upstream_proxy_auth_volume_mount_path(), - "readOnly": true, - }) -} - -fn upstream_proxy_auth_volume_mount_path() -> &'static str { - Path::new(openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH) - .parent() - .and_then(Path::to_str) - .expect("upstream proxy auth path has a parent directory") -} - -fn upstream_proxy_auth_file_name() -> &'static str { - Path::new(openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH) - .file_name() - .and_then(|name| name.to_str()) - .expect("upstream proxy auth path has a UTF-8 file name") -} - -fn has_upstream_proxy_credentials(params: &SandboxPodParams<'_>) -> bool { - params.proxy_auth_secret_name.is_some() && params.proxy_auth_secret_key.is_some() +fn pulling_image_from_kube_message(message: &str) -> Option { + let image = message + .strip_prefix("Pulling image ") + .map(str::trim) + .map(|value| value.trim_matches('"'))?; + (!image.is_empty()).then(|| image.to_string()) } -fn sidecar_state_volume_mount() -> serde_json::Value { - serde_json::json!({ - "name": SIDECAR_STATE_VOLUME_NAME, - "mountPath": SIDECAR_STATE_MOUNT_PATH, - }) +fn pulled_image_label(message: &str) -> String { + extract_image_size(message).map_or_else( + || "Image pulled".to_string(), + |bytes| format!("Image pulled ({})", format_bytes(bytes)), + ) } -fn sidecar_tls_volume_mount() -> serde_json::Value { - serde_json::json!({ - "name": SIDECAR_TLS_VOLUME_NAME, - "mountPath": SIDECAR_TLS_MOUNT_PATH, - }) +fn extract_image_size(message: &str) -> Option { + let size_prefix = "Image size: "; + let start = message.find(size_prefix)? + size_prefix.len(); + let rest = &message[start..]; + let end = rest.find(' ')?; + rest[..end].parse().ok() } -fn copy_log_level_env( - env: &mut Vec, - template_environment: &std::collections::HashMap, - spec_environment: &std::collections::HashMap, -) { - if let Some(value) = spec_environment - .get(openshell_core::sandbox_env::LOG_LEVEL) - .or_else(|| template_environment.get(openshell_core::sandbox_env::LOG_LEVEL)) - { - upsert_env(env, openshell_core::sandbox_env::LOG_LEVEL, value); - } -} +const SANDBOX_RUNTIME_VOLUME_NAME: &str = "openshell-runtime"; +const SANDBOX_STATE_VOLUME_NAME: &str = "openshell-runtime-state"; +const SANDBOX_BOOTSTRAP_VOLUME_NAME: &str = "openshell-sandbox-bootstrap"; +const SANDBOX_POD_IDENTITY_VOLUME_NAME: &str = "openshell-pod-identity"; +const SANDBOX_RUNTIME_MOUNT_PATH: &str = "/.openshell/runtime"; +const SANDBOX_STATE_MOUNT_PATH: &str = "/.openshell/state"; +const SANDBOX_POD_IDENTITY_MOUNT_PATH: &str = "/.openshell/pod-identity"; +const SANDBOX_POD_UID_PATH: &str = "/.openshell/pod-identity/uid"; +const SANDBOX_PROXY_CA_VOLUME_NAME: &str = "openshell-run"; +const SANDBOX_PROXY_CA_MOUNT_PATH: &str = "/run"; +const SANDBOX_BOOTSTRAP_SCHEDULING_GATE: &str = "openshell.ai/bootstrap"; -fn supervisor_sidecar_env( - template_environment: &std::collections::HashMap, - spec_environment: &std::collections::HashMap, +/// Render the workload Pod that runs the `OpenShell` sandbox runtime. +/// +/// The pod receives no gateway credential or endpoint. Its non-root sandbox +/// owns the workload process and seccomp listener. The namespace workload +/// policy permits supervisor Pods to reach its TLS listener; authenticated +/// protocol identity binds the request to the exact supervisor generation. +fn apply_supervisor_sandbox_runtime_boundary( + pod_template: &mut serde_json::Value, params: &SandboxPodParams<'_>, -) -> Vec { - let mut env = Vec::new(); - apply_required_env( - &mut env, - params.sandbox_id, - params.sandbox_name, - params.grpc_endpoint, - "", - !params.client_tls_secret_name.is_empty(), - provider_spiffe_socket_path(params), +) { + let metadata = pod_template + .as_object_mut() + .expect("pod template must be an object") + .entry("metadata") + .or_insert_with(|| serde_json::json!({})); + let labels = metadata + .as_object_mut() + .expect("pod metadata must be an object") + .entry("labels") + .or_insert_with(|| serde_json::json!({})); + let labels = labels + .as_object_mut() + .expect("pod labels must be an object"); + labels.insert( + BOUNDARY_PAIR_LABEL.to_string(), + serde_json::json!(crate::sandbox_runtime::pair_label_value(params.sandbox_id)), ); - if !params.client_tls_secret_name.is_empty() { - upsert_env( - &mut env, - openshell_core::sandbox_env::TLS_CA, - &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/ca.crt"), - ); - upsert_env( - &mut env, - openshell_core::sandbox_env::TLS_CERT, - &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/tls.crt"), - ); - upsert_env( - &mut env, - openshell_core::sandbox_env::TLS_KEY, - &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/tls.key"), - ); - } - copy_log_level_env(&mut env, template_environment, spec_environment); - upsert_env( - &mut env, - openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY, - "sidecar", + labels.insert( + BOUNDARY_ROLE_LABEL.to_string(), + serde_json::json!("workload"), ); - upsert_env( - &mut env, - openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE, - "sidecar-nftables", + + let Some(spec) = pod_template + .get_mut("spec") + .and_then(serde_json::Value::as_object_mut) + else { + return; + }; + spec.insert("hostNetwork".to_string(), serde_json::json!(false)); + spec.insert("hostPID".to_string(), serde_json::json!(false)); + spec.insert("hostIPC".to_string(), serde_json::json!(false)); + spec.insert( + "shareProcessNamespace".to_string(), + serde_json::json!(false), ); - upsert_env( - &mut env, - openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET, - SIDECAR_CONTROL_SOCKET, + spec.insert("dnsPolicy".to_string(), serde_json::json!("None")); + spec.insert( + "schedulingGates".to_string(), + serde_json::json!([{"name": SANDBOX_BOOTSTRAP_SCHEDULING_GATE}]), ); - upsert_env( - &mut env, - openshell_core::sandbox_env::SSH_SOCKET_PATH, - SIDECAR_SSH_SOCKET_FILE, + spec.insert( + "dnsConfig".to_string(), + serde_json::json!({ + "nameservers": ["127.0.0.53"], + "options": [ + {"name": "ndots", "value": "5"}, + {"name": "timeout", "value": "2"}, + {"name": "attempts", "value": "2"} + ] + }), ); - upsert_env( - &mut env, - openshell_core::sandbox_env::PROXY_TLS_DIR, - SIDECAR_TLS_MOUNT_PATH, + spec.insert( + "securityContext".to_string(), + serde_json::json!({ + "runAsUser": params.sandbox_uid, + "runAsGroup": params.sandbox_gid, + "runAsNonRoot": true, + "fsGroup": params.sandbox_gid, + "fsGroupChangePolicy": "OnRootMismatch", + "supplementalGroups": [], + "supplementalGroupsPolicy": "Strict", + "seccompProfile": {"type": "RuntimeDefault"}, + "sysctls": [{"name": "net.ipv4.ip_unprivileged_port_start", "value": "0"}] + }), ); - apply_resolved_identity_env(&mut env, params.sandbox_uid, params.sandbox_gid); - if !params.process_binary_aware_network_policy { - upsert_env( - &mut env, - openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY, - "relaxed", - ); - } - env -} - -fn supervisor_sidecar_container( - template_environment: &std::collections::HashMap, - spec_environment: &std::collections::HashMap, - params: &SandboxPodParams<'_>, -) -> serde_json::Value { - let proxy_uid = effective_sidecar_proxy_uid(params); - let capabilities = if params.process_binary_aware_network_policy { + let volumes = spec + .entry("volumes") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut() + .expect("pod volumes must be an array"); + volumes.retain(|volume| { + !matches!( + volume.get("name").and_then(serde_json::Value::as_str), + Some( + "openshell-sa-token" + | "openshell-client-tls" + | "spiffe-workload-api" + | SANDBOX_RUNTIME_VOLUME_NAME + | SANDBOX_STATE_VOLUME_NAME + | SANDBOX_BOOTSTRAP_VOLUME_NAME + | SANDBOX_POD_IDENTITY_VOLUME_NAME + | SANDBOX_PROXY_CA_VOLUME_NAME + ) + ) + }); + volumes.extend([ + serde_json::json!({"name": SANDBOX_RUNTIME_VOLUME_NAME, "emptyDir": {"medium": "Memory"}}), + serde_json::json!({"name": SANDBOX_STATE_VOLUME_NAME, "emptyDir": {"medium": "Memory"}}), serde_json::json!({ - "drop": ["ALL"], - "add": ["SYS_PTRACE", "DAC_READ_SEARCH"] - }) - } else { + "name": SANDBOX_BOOTSTRAP_VOLUME_NAME, + "secret": {"secretName": params.sandbox_secret_name, "defaultMode": 0o440} + }), serde_json::json!({ - "drop": ["ALL"] - }) - }; - let mut container = serde_json::json!({ - "name": SUPERVISOR_NETWORK_SIDECAR_NAME, - "image": params.supervisor_image, - "command": [ - SUPERVISOR_IMAGE_BINARY_PATH, - "--mode=network", - ], - "env": supervisor_sidecar_env(template_environment, spec_environment, params), - "securityContext": { - "runAsUser": proxy_uid, - "runAsGroup": params.sandbox_gid, - "runAsNonRoot": proxy_uid != 0, - "allowPrivilegeEscalation": false, - "capabilities": capabilities - }, - "volumeMounts": [ - sidecar_state_volume_mount(), - sidecar_tls_volume_mount(), - { - "name": "openshell-sa-token", - "mountPath": "/var/run/secrets/openshell", - "readOnly": true + "name": SANDBOX_POD_IDENTITY_VOLUME_NAME, + "downwardAPI": { + "items": [{"path": "uid", "fieldRef": {"fieldPath": "metadata.uid"}}] } - ] - }); - container["command"] + }), + serde_json::json!({ + "name": SANDBOX_PROXY_CA_VOLUME_NAME, + "emptyDir": {"medium": "Memory"} + }), + ]); + + let init_containers = spec + .entry("initContainers") + .or_insert_with(|| serde_json::json!([])) .as_array_mut() - .expect("network supervisor command is an array") - .extend( - upstream_proxy_cli_args(params) - .into_iter() - .map(serde_json::Value::String), - ); - if let Some(policy) = params.supervisor_image_pull_policy { - container["imagePullPolicy"] = serde_json::json!(policy); - } - if params.provider_spiffe_enabled { - container["volumeMounts"] - .as_array_mut() - .expect("volumeMounts is an array") - .push(serde_json::json!({ - "name": SPIFFE_WORKLOAD_API_VOLUME_NAME, - "mountPath": spiffe_socket_mount_path(params.provider_spiffe_workload_api_socket_path), - "readOnly": true, - })); - } - if has_upstream_proxy_credentials(params) { - container["volumeMounts"] - .as_array_mut() - .expect("volumeMounts is an array") - .push(upstream_proxy_auth_volume_mount()); - } - if let Some(profile) = params.app_armor_profile { - container["securityContext"]["appArmorProfile"] = app_armor_profile_to_k8s(profile); - } - container -} - -fn effective_sidecar_proxy_uid(params: &SandboxPodParams<'_>) -> u32 { - if params.process_binary_aware_network_policy { - BINARY_AWARE_SIDECAR_PROXY_UID - } else { - params.proxy_uid - } -} - -fn supervisor_network_init_container(params: &SandboxPodParams<'_>) -> serde_json::Value { - let proxy_uid = effective_sidecar_proxy_uid(params); - let mut container = serde_json::json!({ - "name": SUPERVISOR_NETWORK_INIT_CONTAINER_NAME, - "image": params.supervisor_image, - "command": [ - SUPERVISOR_IMAGE_BINARY_PATH, - "--mode=network-init", - "--proxy-uid", - proxy_uid.to_string(), - "--proxy-gid", - params.sandbox_gid.to_string(), - "--sidecar-state-dir", - SIDECAR_STATE_MOUNT_PATH, - "--sidecar-tls-dir", - SIDECAR_TLS_MOUNT_PATH, - ], + .expect("pod init containers must be an array"); + let mut bootstrap = serde_json::json!({ + "name": "openshell-sandbox-bootstrap", + "image": params.sandbox_runtime_image, + "command": ["/openshell-sandbox", "bootstrap"], "securityContext": { - "runAsUser": 0, + "runAsUser": params.sandbox_uid, + "runAsGroup": params.sandbox_gid, + "runAsNonRoot": true, + "readOnlyRootFilesystem": true, "allowPrivilegeEscalation": false, - "capabilities": { - "drop": ["ALL"], - "add": ["NET_ADMIN", "NET_RAW", "CHOWN", "FOWNER"] - } + "capabilities": {"drop": ["ALL"]} }, "volumeMounts": [ - sidecar_state_volume_mount(), - sidecar_tls_volume_mount(), + {"name": SANDBOX_BOOTSTRAP_VOLUME_NAME, "mountPath": crate::sandbox_runtime::SANDBOX_BOOTSTRAP_INPUT_PATH, "readOnly": true}, + {"name": SANDBOX_RUNTIME_VOLUME_NAME, "mountPath": SANDBOX_RUNTIME_MOUNT_PATH}, + {"name": SANDBOX_STATE_VOLUME_NAME, "mountPath": SANDBOX_STATE_MOUNT_PATH} ] }); - if let Some(policy) = params.supervisor_image_pull_policy { - container["imagePullPolicy"] = serde_json::json!(policy); - } - if !params.client_tls_secret_name.is_empty() { - container["volumeMounts"] - .as_array_mut() - .expect("volumeMounts is an array") - .push(serde_json::json!({ - "name": "openshell-client-tls", - "mountPath": openshell_core::container_paths::CLIENT_TLS_DIR, - "readOnly": true - })); - } - if let Some(profile) = params.app_armor_profile { - container["securityContext"]["appArmorProfile"] = app_armor_profile_to_k8s(profile); - } - container -} - -fn apply_supervisor_sidecar_topology( - pod_template: &mut serde_json::Value, - template_environment: &std::collections::HashMap, - spec_environment: &std::collections::HashMap, - params: &SandboxPodParams<'_>, -) { - let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { - return; - }; - - let pod_security_context = spec - .entry("securityContext") - .or_insert_with(|| serde_json::json!({})); - if let Some(sc) = pod_security_context.as_object_mut() { - sc.insert("fsGroup".to_string(), serde_json::json!(params.sandbox_gid)); + if let Some(policy) = params.sandbox_runtime_image_pull_policy { + bootstrap["imagePullPolicy"] = serde_json::json!(policy.as_kubernetes_str()); } + init_containers.push(bootstrap); - spec.insert("shareProcessNamespace".to_string(), serde_json::json!(true)); - - apply_supervisor_binary_source( - spec, - params.supervisor_image, - params.supervisor_image_pull_policy, - params.supervisor_sideload_method, + let containers = spec + .get_mut("containers") + .and_then(serde_json::Value::as_array_mut) + .expect("pod containers must be an array"); + let index = containers + .iter() + .position(|container| { + container.get("name").and_then(serde_json::Value::as_str) == Some("agent") + }) + .unwrap_or(0); + let container = containers[index] + .as_object_mut() + .expect("agent container must be an object"); + container.insert( + "command".to_string(), + serde_json::json!([ + format!("{SANDBOX_RUNTIME_MOUNT_PATH}/openshell-sandbox"), + "--bootstrap", + BOUNDARY_CONFIG_PATH, + ]), ); - - let volumes = spec - .entry("volumes") + container.insert( + "securityContext".to_string(), + serde_json::json!({ + "runAsUser": params.sandbox_uid, + "runAsGroup": params.sandbox_gid, + "runAsNonRoot": true, + "allowPrivilegeEscalation": false, + "capabilities": {"drop": ["ALL"]} + }), + ); + container.insert( + "ports".to_string(), + serde_json::json!([{ + "name": "sandbox-control", + "containerPort": params.boundary_port, + "protocol": "TCP" + }]), + ); + let mounts = container + .entry("volumeMounts") .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(volumes) = volumes { - volumes.push(serde_json::json!({ - "name": SIDECAR_STATE_VOLUME_NAME, - "emptyDir": {} - })); - volumes.push(serde_json::json!({ - "name": SIDECAR_TLS_VOLUME_NAME, - "emptyDir": {} - })); - } - - let init_containers = spec - .entry("initContainers") + .as_array_mut() + .expect("agent volume mounts must be an array"); + mounts.retain(|mount| { + !matches!( + mount.get("name").and_then(serde_json::Value::as_str), + Some( + "openshell-sa-token" + | "openshell-client-tls" + | "spiffe-workload-api" + | SANDBOX_BOOTSTRAP_VOLUME_NAME + ) + ) + }); + mounts.extend([ + serde_json::json!({"name": SANDBOX_RUNTIME_VOLUME_NAME, "mountPath": SANDBOX_RUNTIME_MOUNT_PATH, "readOnly": true}), + serde_json::json!({"name": SANDBOX_STATE_VOLUME_NAME, "mountPath": SANDBOX_STATE_MOUNT_PATH}), + serde_json::json!({"name": SANDBOX_POD_IDENTITY_VOLUME_NAME, "mountPath": SANDBOX_POD_IDENTITY_MOUNT_PATH, "readOnly": true}), + serde_json::json!({"name": SANDBOX_PROXY_CA_VOLUME_NAME, "mountPath": SANDBOX_PROXY_CA_MOUNT_PATH}), + ]); + let env = container + .entry("env") .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(init_containers) = init_containers { - init_containers.push(supervisor_network_init_container(params)); - } - - let Some(containers) = spec.get_mut("containers").and_then(|v| v.as_array_mut()) else { - return; - }; - - let target_index = containers - .iter() - .position(|c| c.get("name").and_then(|v| v.as_str()) == Some("agent")) - .unwrap_or(0); - - if let Some(container) = containers - .get_mut(target_index) - .and_then(|v| v.as_object_mut()) - { - container.insert( - "command".to_string(), - serde_json::json!([ - format!("{}/openshell-sandbox", SUPERVISOR_MOUNT_PATH), - "--mode=process", - "--workdir", - driver_mounts::DEFAULT_WORKSPACE_ROOT - ]), - ); - - let security_context = container - .entry("securityContext") - .or_insert_with(|| serde_json::json!({})); - if let Some(sc) = security_context.as_object_mut() { - sc.insert( - "runAsUser".to_string(), - serde_json::json!(params.sandbox_uid), - ); - sc.insert( - "runAsGroup".to_string(), - serde_json::json!(params.sandbox_gid), - ); - sc.insert("runAsNonRoot".to_string(), serde_json::json!(true)); - sc.insert( - "allowPrivilegeEscalation".to_string(), - serde_json::json!(false), - ); - sc.insert( - "capabilities".to_string(), - serde_json::json!({ - "drop": ["ALL"] - }), - ); - } - - let volume_mounts = container - .entry("volumeMounts") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(volume_mounts) = volume_mounts { - remove_volume_mount(volume_mounts, "openshell-sa-token"); - remove_volume_mount(volume_mounts, "openshell-client-tls"); - remove_volume_mount(volume_mounts, SPIFFE_WORKLOAD_API_VOLUME_NAME); - volume_mounts.push(supervisor_volume_mount()); - volume_mounts.push(sidecar_state_volume_mount()); - volume_mounts.push(sidecar_tls_volume_mount()); - } - - let env = container - .entry("env") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(env) = env { - remove_env(env, openshell_core::sandbox_env::ENDPOINT); - remove_env(env, openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME); - remove_env(env, openshell_core::sandbox_env::TLS_CA); - remove_env(env, openshell_core::sandbox_env::TLS_CERT); - remove_env(env, openshell_core::sandbox_env::TLS_KEY); - remove_env(env, openshell_core::sandbox_env::SANDBOX_TOKEN); - remove_env(env, openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); - remove_env(env, openshell_core::sandbox_env::K8S_SA_TOKEN_FILE); - remove_env( - env, - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, - ); - upsert_env( - env, - openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY, - "sidecar", - ); - upsert_env( - env, - openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE, - "sidecar-nftables", - ); - upsert_env( - env, - openshell_core::sandbox_env::SSH_SOCKET_PATH, - SIDECAR_SSH_SOCKET_FILE, - ); - upsert_env( - env, - openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET, - SIDECAR_CONTROL_SOCKET, - ); - upsert_env( - env, - openshell_core::sandbox_env::PROXY_TLS_DIR, - SIDECAR_TLS_MOUNT_PATH, - ); - apply_resolved_identity_env(env, params.sandbox_uid, params.sandbox_gid); - } + .as_array_mut() + .expect("agent environment must be an array"); + for key in [ + openshell_core::sandbox_env::ENDPOINT, + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME, + openshell_core::sandbox_env::TLS_CA, + openshell_core::sandbox_env::TLS_CERT, + openshell_core::sandbox_env::TLS_KEY, + openshell_core::sandbox_env::SANDBOX_TOKEN, + openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, + openshell_core::sandbox_env::K8S_SA_TOKEN_FILE, + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, + openshell_core::sandbox_env::MAIN_PROCESS_SPEC, + ] { + remove_env(env, key); } - - containers.push(supervisor_sidecar_container( - template_environment, - spec_environment, - params, - )); + apply_resolved_identity_env(env, params.sandbox_uid, params.sandbox_gid); } /// Apply workspace persistence transforms to an already-built pod template. @@ -3397,8 +5419,9 @@ fn apply_supervisor_sidecar_topology( fn apply_workspace_persistence( pod_template: &mut serde_json::Value, image: &str, - image_pull_policy: Option<&str>, - sandbox_gid: u32, + image_pull_policy: Option, + sandbox_gid: Option, + workspace_owner: Option<(u32, u32)>, ) { let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { return; @@ -3406,11 +5429,13 @@ fn apply_workspace_persistence( // fsGroup is a pod-level field — it instructs kubelet to chown mounted // volumes to this GID. It is invalid at the container securityContext level. - let pod_sc = spec - .entry("securityContext") - .or_insert_with(|| serde_json::json!({})); - if let Some(pod_sc_obj) = pod_sc.as_object_mut() { - pod_sc_obj.insert("fsGroup".to_string(), serde_json::json!(sandbox_gid)); + if let Some(sandbox_gid) = sandbox_gid { + let pod_sc = spec + .entry("securityContext") + .or_insert_with(|| serde_json::json!({})); + if let Some(pod_sc_obj) = pod_sc.as_object_mut() { + pod_sc_obj.insert("fsGroup".to_string(), serde_json::json!(sandbox_gid)); + } } // 1. Add workspace volume mount to the agent container @@ -3475,20 +5500,38 @@ fn apply_workspace_persistence( fi" ); - let mut init_spec = serde_json::json!({ - "name": WORKSPACE_INIT_CONTAINER_NAME, - "image": image, - "command": ["sh", "-c", copy_cmd], - "securityContext": { - "runAsUser": 0, - }, - "volumeMounts": [{ - "name": WORKSPACE_VOLUME_NAME, - "mountPath": WORKSPACE_INIT_MOUNT_PATH - }] - }); + let mut init_spec = if let Some((uid, gid)) = workspace_owner { + serde_json::json!({ + "name": WORKSPACE_INIT_CONTAINER_NAME, + "image": image, + "command": [format!("{SANDBOX_RUNTIME_MOUNT_PATH}/openshell-sandbox"), "seed-workspace"], + "securityContext": { + "runAsUser": uid, + "runAsGroup": gid, + "runAsNonRoot": true, + "readOnlyRootFilesystem": true, + "allowPrivilegeEscalation": false, + "capabilities": {"drop": ["ALL"]} + }, + "volumeMounts": [ + {"name": WORKSPACE_VOLUME_NAME, "mountPath": WORKSPACE_INIT_MOUNT_PATH}, + {"name": SANDBOX_RUNTIME_VOLUME_NAME, "mountPath": SANDBOX_RUNTIME_MOUNT_PATH, "readOnly": true} + ] + }) + } else { + serde_json::json!({ + "name": WORKSPACE_INIT_CONTAINER_NAME, + "image": image, + "command": ["sh", "-c", copy_cmd], + "securityContext": {"runAsUser": 0}, + "volumeMounts": [{ + "name": WORKSPACE_VOLUME_NAME, + "mountPath": WORKSPACE_INIT_MOUNT_PATH + }] + }) + }; if let Some(policy) = image_pull_policy { - init_spec["imagePullPolicy"] = serde_json::json!(policy); + init_spec["imagePullPolicy"] = serde_json::json!(policy.as_kubernetes_str()); } init_containers.push(init_spec); } @@ -3535,41 +5578,24 @@ fn default_workspace_volume_claim_templates( #[allow(clippy::struct_excessive_bools)] struct SandboxPodParams<'a> { default_image: &'a str, - image_pull_policy: Option<&'a str>, + image_pull_policy: Option, image_pull_secrets: &'a [String], - supervisor_image: &'a str, - supervisor_image_pull_policy: Option<&'a str>, - supervisor_sideload_method: SupervisorSideloadMethod, - topology: SupervisorTopology, - proxy_uid: u32, - process_binary_aware_network_policy: bool, - https_proxy: Option<&'a str>, - no_proxy: Option<&'a str>, - proxy_auth_secret_name: Option<&'a str>, - proxy_auth_secret_key: Option<&'a str>, - proxy_auth_allow_insecure: bool, - proxy_connect_by_hostname: bool, + sandbox_runtime_image: &'a str, + sandbox_runtime_image_pull_policy: Option, service_account_name: &'a str, sandbox_id: &'a str, - sandbox_name: &'a str, - grpc_endpoint: &'a str, - ssh_socket_path: &'a str, - client_tls_secret_name: &'a str, - host_gateway_ip: &'a str, enable_user_namespaces: bool, - app_armor_profile: Option<&'a AppArmorProfile>, workspace_default_storage_size: &'a str, workspace_storage_class: &'a str, default_runtime_class_name: &'a str, - /// Lifetime (seconds) of the projected `ServiceAccount` token used - /// for the bootstrap `IssueSandboxToken` exchange. - sa_token_ttl_secs: i64, - provider_spiffe_enabled: bool, - provider_spiffe_workload_api_socket_path: &'a str, /// Resolved sandbox UID for supervisor `runAsUser` and env var. sandbox_uid: u32, /// Resolved sandbox GID for PVC init container operations. sandbox_gid: u32, + /// TLS listener port exposed only to the paired supervisor Pod. + boundary_port: u16, + /// Immutable Secret name for this workload Pod generation. + sandbox_secret_name: &'a str, } impl Default for SandboxPodParams<'_> { @@ -3578,51 +5604,22 @@ impl Default for SandboxPodParams<'_> { default_image: "", image_pull_policy: None, image_pull_secrets: &[], - supervisor_image: "", - supervisor_image_pull_policy: None, - supervisor_sideload_method: SupervisorSideloadMethod::default(), - topology: SupervisorTopology::default(), - proxy_uid: DEFAULT_PROXY_UID, - process_binary_aware_network_policy: true, - https_proxy: None, - no_proxy: None, - proxy_auth_secret_name: None, - proxy_auth_secret_key: None, - proxy_auth_allow_insecure: false, - proxy_connect_by_hostname: false, + sandbox_runtime_image: "", + sandbox_runtime_image_pull_policy: None, service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, sandbox_id: "", - sandbox_name: "", - grpc_endpoint: "", - ssh_socket_path: "", - client_tls_secret_name: "", - host_gateway_ip: "", enable_user_namespaces: false, - app_armor_profile: None, workspace_default_storage_size: DEFAULT_WORKSPACE_STORAGE_SIZE, workspace_storage_class: "", default_runtime_class_name: "", - sa_token_ttl_secs: 3600, - provider_spiffe_enabled: false, - provider_spiffe_workload_api_socket_path: "", sandbox_uid: DEFAULT_SANDBOX_UID, sandbox_gid: DEFAULT_SANDBOX_UID, + boundary_port: 5500, + sandbox_secret_name: "os-sandbox-test-generation", } } } -fn validate_sidecar_proxy_identity( - params: &SandboxPodParams<'_>, -) -> Result<(), KubernetesDriverError> { - if params.topology == SupervisorTopology::Sidecar && params.proxy_uid == params.sandbox_uid { - return Err(KubernetesDriverError::Precondition(format!( - "proxy_uid ({}) must not match sandbox_uid ({}) in sidecar topology", - params.proxy_uid, params.sandbox_uid - ))); - } - Ok(()) -} - fn spec_pod_env(spec: Option<&SandboxSpec>) -> std::collections::HashMap { let mut env = spec.map_or_else(Default::default, |s| s.environment.clone()); if let Some(s) = spec.filter(|s| !s.log_level.is_empty()) { @@ -3636,22 +5633,15 @@ fn spec_pod_env(spec: Option<&SandboxSpec>) -> std::collections::HashMap, - provider_spiffe_workload_api_socket_path: Option<&str>, ) -> Result { let config = spec .and_then(|spec| spec.template.as_ref()) .map(KubernetesSandboxDriverConfig::from_template) .transpose()? .unwrap_or_default(); - let mut protected_paths = KUBERNETES_DRIVER_PROTECTED_MOUNT_PATHS.to_vec(); - let provider_spiffe_mount_path; - if let Some(socket_path) = provider_spiffe_workload_api_socket_path { - provider_spiffe_mount_path = spiffe_socket_mount_path(socket_path); - protected_paths.push(&provider_spiffe_mount_path); - } validate_kubernetes_protected_path_conflicts( &config.containers.agent.volume_mounts, - &protected_paths, + KUBERNETES_DRIVER_PROTECTED_MOUNT_PATHS, )?; Ok(config) } @@ -3660,8 +5650,7 @@ fn sandbox_to_k8s_spec( spec: Option<&SandboxSpec>, params: &SandboxPodParams<'_>, ) -> Result { - let driver_config = - kubernetes_driver_config_for_spec(spec, provider_spiffe_socket_path(params))?; + let driver_config = kubernetes_driver_config_for_spec(spec)?; let mut root = serde_json::Map::new(); // Determine early whether OpenShell should inject its default workspace @@ -3681,7 +5670,6 @@ fn sandbox_to_k8s_spec( template, driver_gpu_requirements(spec.resource_requirements.as_ref()), &pod_env, - Some(spec), &driver_config, inject_workspace, params, @@ -3715,7 +5703,6 @@ fn sandbox_to_k8s_spec( &SandboxTemplate::default(), driver_gpu_requirements(spec.and_then(|s| s.resource_requirements.as_ref())), &pod_env, - spec, &driver_config, inject_workspace, params, @@ -3743,7 +5730,6 @@ fn sandbox_template_to_k8s( template, gpu_requirements.as_ref(), spec_environment, - None, &driver_config, inject_workspace, params, @@ -3764,7 +5750,6 @@ fn sandbox_template_to_k8s_with_gpu_requirements( template, gpu_requirements, spec_environment, - None, &driver_config, inject_workspace, params, @@ -3775,29 +5760,16 @@ fn sandbox_template_to_k8s_with_validated_config( template: &SandboxTemplate, gpu_requirements: Option<&GpuResourceRequirements>, spec_environment: &std::collections::HashMap, - sandbox_spec: Option<&openshell_core::proto::compute::v1::DriverSandboxSpec>, driver_config: &KubernetesSandboxDriverConfig, inject_workspace: bool, params: &SandboxPodParams<'_>, ) -> serde_json::Value { let mut metadata = serde_json::Map::new(); - let mut pod_labels = template + let pod_labels = template .labels .iter() .map(|(key, value)| (key.clone(), serde_json::Value::String(value.clone()))) .collect::>(); - if params.provider_spiffe_enabled { - pod_labels.insert( - LABEL_MANAGED_BY.to_string(), - serde_json::Value::String(LABEL_MANAGED_BY_VALUE.to_string()), - ); - if !params.sandbox_id.is_empty() { - pod_labels.insert( - LABEL_SANDBOX_ID.to_string(), - serde_json::Value::String(params.sandbox_id.to_string()), - ); - } - } if !pod_labels.is_empty() { metadata.insert("labels".to_string(), serde_json::Value::Object(pod_labels)); } @@ -3905,75 +5877,24 @@ fn sandbox_template_to_k8s_with_validated_config( if !image.is_empty() { container.insert("image".to_string(), serde_json::json!(image)); if let Some(policy) = params.image_pull_policy { - container.insert("imagePullPolicy".to_string(), serde_json::json!(policy)); + container.insert( + "imagePullPolicy".to_string(), + serde_json::json!(policy.as_kubernetes_str()), + ); } } - // Build environment variables - start with OpenShell-required vars - let env = build_env_list( - None, - &template.environment, - spec_environment, - sandbox_spec, - params.sandbox_id, - params.sandbox_name, - params.grpc_endpoint, - params.ssh_socket_path, - !params.client_tls_secret_name.is_empty(), - provider_spiffe_socket_path(params), - ); + let env = build_sandbox_env(&template.environment, spec_environment); container.insert("env".to_string(), serde_json::Value::Array(env)); - let mut capabilities: Vec<&str> = vec!["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE", "SYSLOG"]; - if use_user_namespaces { - // In a user namespace the bounding set is reset. SETUID/SETGID are - // needed for the supervisor to drop privileges to the sandbox user. - // DAC_READ_SEARCH is needed for cross-UID /proc//fd/ access - // for process identity resolution in network policy enforcement. - capabilities.extend(["SETUID", "SETGID", "DAC_READ_SEARCH"]); - } - let mut security_context = serde_json::json!({ - "capabilities": { - "add": capabilities - } - }); - if let Some(profile) = params.app_armor_profile { - security_context["appArmorProfile"] = app_armor_profile_to_k8s(profile); - } - container.insert("securityContext".to_string(), security_context); - - // Mount client TLS secret for mTLS to the server. Gateway identity uses - // the projected ServiceAccount bootstrap token. Provider token grants may - // additionally mount the SPIFFE Workload API socket. - let mut volume_mounts: Vec = Vec::new(); - if !params.client_tls_secret_name.is_empty() { - volume_mounts.push(serde_json::json!({ - "name": CLIENT_TLS_VOLUME_NAME, - "mountPath": openshell_core::container_paths::CLIENT_TLS_DIR, - "readOnly": true - })); - } - if params.provider_spiffe_enabled { - volume_mounts.push(serde_json::json!({ - "name": SPIFFE_WORKLOAD_API_VOLUME_NAME, - "mountPath": spiffe_socket_mount_path(params.provider_spiffe_workload_api_socket_path), - "readOnly": true, - })); - } - volume_mounts.push(serde_json::json!({ - "name": SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, - "mountPath": SERVICE_ACCOUNT_TOKEN_MOUNT_PATH, - "readOnly": true, - })); - volume_mounts.extend( - driver_config - .containers - .agent - .volume_mounts - .iter() - .map(kubernetes_driver_volume_mount_to_k8s), - ); + let volume_mounts = driver_config + .containers + .agent + .volume_mounts + .iter() + .map(kubernetes_driver_volume_mount_to_k8s) + .collect::>(); container.insert( "volumeMounts".to_string(), serde_json::Value::Array(volume_mounts), @@ -3988,100 +5909,13 @@ fn sandbox_template_to_k8s_with_validated_config( serde_json::Value::Array(vec![serde_json::Value::Object(container)]), ); - // Add TLS secret volume. Combined mode uses mode 0400 because the - // supervisor starts as root and drops privileges before running workload - // children. Sidecar mode keeps the process supervisor non-root, so it uses - // pod fsGroup + 0440 to preserve gateway session and SSH control behavior. - let mut volumes: Vec = Vec::new(); - if !params.client_tls_secret_name.is_empty() { - let client_tls_default_mode = match params.topology { - SupervisorTopology::Combined => 0o400, - SupervisorTopology::Sidecar => 0o440, - }; - volumes.push(serde_json::json!({ - "name": CLIENT_TLS_VOLUME_NAME, - "secret": { - "secretName": params.client_tls_secret_name, - "defaultMode": client_tls_default_mode - } - })); - } - if has_upstream_proxy_credentials(params) { - let secret_name = params - .proxy_auth_secret_name - .expect("complete proxy credential reference has a Secret name"); - let secret_key = params - .proxy_auth_secret_key - .expect("complete proxy credential reference has a Secret key"); - // The credential volume is mounted only into the container that runs - // network supervision. Sidecar mode uses the pod fsGroup already - // required for its non-root network supervisor. - let default_mode = match params.topology { - SupervisorTopology::Combined => 0o400, - SupervisorTopology::Sidecar => 0o440, - }; - volumes.push(serde_json::json!({ - "name": UPSTREAM_PROXY_AUTH_VOLUME_NAME, - "secret": { - "secretName": secret_name, - "defaultMode": default_mode, - "items": [{ - "key": secret_key, - "path": upstream_proxy_auth_file_name(), - }] - } - })); - } - if params.provider_spiffe_enabled { - volumes.push(serde_json::json!({ - "name": SPIFFE_WORKLOAD_API_VOLUME_NAME, - "csi": { - "driver": "csi.spiffe.io", - "readOnly": true - } - })); - } - // Projected ServiceAccountToken volume — kubelet writes a short-lived - // audience-bound JWT into /var/run/secrets/openshell/token and rotates - // it automatically. The supervisor exchanges this for a gateway-minted - // JWT via `IssueSandboxToken` once at startup. In sidecar topology both - // supervisor containers run with the sandbox GID and need group-read access. - let sa_token_default_mode = match params.topology { - SupervisorTopology::Combined => 0o400, - SupervisorTopology::Sidecar => 0o440, - }; - volumes.push(serde_json::json!({ - "name": SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, - "projected": { - "sources": [{ - "serviceAccountToken": { - "audience": "openshell-gateway", - "expirationSeconds": params.sa_token_ttl_secs, - "path": "token" - } - }], - "defaultMode": sa_token_default_mode - } - })); - volumes.extend( - driver_config - .volumes - .iter() - .map(kubernetes_driver_volume_to_k8s), - ); + let volumes = driver_config + .volumes + .iter() + .map(kubernetes_driver_volume_to_k8s) + .collect::>(); spec.insert("volumes".to_string(), serde_json::Value::Array(volumes)); - // Add hostAliases so sandbox pods can reach the Docker host. - if !params.host_gateway_ip.is_empty() { - spec.insert( - "hostAliases".to_string(), - serde_json::json!([{ - "ip": params.host_gateway_ip, - "hostnames": ["host.docker.internal", "host.openshell.internal"] - }]), - ); - } - let mut template_value = serde_json::Map::new(); if !metadata.is_empty() { template_value.insert("metadata".to_string(), serde_json::Value::Object(metadata)); @@ -4090,19 +5924,7 @@ fn sandbox_template_to_k8s_with_validated_config( let mut result = serde_json::Value::Object(template_value); - match params.topology { - SupervisorTopology::Combined => { - apply_supervisor_sideload_with_params(&mut result, params); - } - SupervisorTopology::Sidecar => { - apply_supervisor_sidecar_topology( - &mut result, - &template.environment, - spec_environment, - params, - ); - } - } + apply_supervisor_sandbox_runtime_boundary(&mut result, params); // Inject workspace persistence (init container + PVC volume mount) so // that /sandbox data survives pod rescheduling. Skipped when the user @@ -4112,7 +5934,8 @@ fn sandbox_template_to_k8s_with_validated_config( &mut result, image, params.image_pull_policy, - params.sandbox_gid, + None, + Some((params.sandbox_uid, params.sandbox_gid)), ); } @@ -4205,16 +6028,6 @@ fn image_pull_secret_refs(secrets: &[String]) -> Vec { .collect() } -fn app_armor_profile_to_k8s(profile: &AppArmorProfile) -> serde_json::Value { - let mut value = serde_json::json!({ - "type": profile.kubernetes_type() - }); - if let Some(localhost_profile) = profile.localhost_profile() { - value["localhostProfile"] = serde_json::json!(localhost_profile); - } - value -} - fn container_resources( template: &SandboxTemplate, gpu_requirements: Option<&GpuResourceRequirements>, @@ -4279,144 +6092,22 @@ fn apply_gpu_limit(resources: &mut serde_json::Value, quantity: &str) { limits_obj.insert(GPU_RESOURCE_NAME.to_string(), serde_json::json!(quantity)); } -#[allow(clippy::too_many_arguments)] -fn build_env_list( - existing_env: Option<&Vec>, +fn build_sandbox_env( template_environment: &std::collections::HashMap, spec_environment: &std::collections::HashMap, - sandbox_spec: Option<&openshell_core::proto::compute::v1::DriverSandboxSpec>, - sandbox_id: &str, - sandbox_name: &str, - grpc_endpoint: &str, - ssh_socket_path: &str, - tls_enabled: bool, - provider_spiffe_socket_path: Option<&str>, ) -> Vec { - let mut env = existing_env.cloned().unwrap_or_default(); - apply_env_map(&mut env, template_environment); - apply_env_map(&mut env, spec_environment); - let mut user_env = template_environment.clone(); - user_env.extend(spec_environment.clone()); - if !user_env.is_empty() - && let Ok(json) = serde_json::to_string(&user_env) - { - upsert_env( - &mut env, - openshell_core::sandbox_env::USER_ENVIRONMENT, - &json, - ); + let mut env = Vec::new(); + for (name, value) in template_environment.iter().chain(spec_environment) { + if !name.starts_with("OPENSHELL_") || name == openshell_core::sandbox_env::LOG_LEVEL { + upsert_env(&mut env, name, value); + } } - let main_process = - openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(sandbox_spec) - .expect("main process config serialization cannot fail"); upsert_env( &mut env, - openshell_core::sandbox_env::MAIN_PROCESS_SPEC, - &main_process, - ); - apply_required_env( - &mut env, - sandbox_id, - sandbox_name, - grpc_endpoint, - ssh_socket_path, - tls_enabled, - provider_spiffe_socket_path, - ); - env -} - -fn apply_env_map( - env: &mut Vec, - values: &std::collections::HashMap, -) { - for (key, value) in values { - upsert_env(env, key, value); - } -} - -// Required env vars are passed individually for clarity at call sites; grouping into a struct -// would not improve readability for this internal helper. -fn apply_required_env( - env: &mut Vec, - sandbox_id: &str, - sandbox_name: &str, - grpc_endpoint: &str, - ssh_socket_path: &str, - tls_enabled: bool, - provider_spiffe_socket_path: Option<&str>, -) { - upsert_env(env, openshell_core::sandbox_env::SANDBOX_ID, sandbox_id); - upsert_env(env, openshell_core::sandbox_env::SANDBOX, sandbox_name); - upsert_env(env, openshell_core::sandbox_env::ENDPOINT, grpc_endpoint); - upsert_env( - env, openshell_core::sandbox_env::TELEMETRY_ENABLED, openshell_core::telemetry::enabled_env_value(), ); - // Runtime capabilities are driver-owned. Kubernetes topologies do not yet - // provide the complete policy DNS and transparent TCP substrate. - upsert_env( - env, - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, - "", - ); - if !ssh_socket_path.is_empty() { - upsert_env( - env, - openshell_core::sandbox_env::SSH_SOCKET_PATH, - ssh_socket_path, - ); - } - // TLS cert paths for sandbox-to-server mTLS. Only set when TLS is enabled - // and the client TLS secret is mounted into the sandbox pod. - if tls_enabled { - upsert_env( - env, - openshell_core::sandbox_env::TLS_CA, - "/etc/openshell-tls/client/ca.crt", - ); - upsert_env( - env, - openshell_core::sandbox_env::TLS_CERT, - "/etc/openshell-tls/client/tls.crt", - ); - upsert_env( - env, - openshell_core::sandbox_env::TLS_KEY, - "/etc/openshell-tls/client/tls.key", - ); - } - // Projected ServiceAccount token written by kubelet (see the volume - // definition in `sandbox_template_to_k8s`). The supervisor reads this - // and exchanges it for a gateway-minted JWT via `IssueSandboxToken`. - upsert_env( - env, - openshell_core::sandbox_env::K8S_SA_TOKEN_FILE, - "/var/run/secrets/openshell/token", - ); - if let Some(socket_path) = provider_spiffe_socket_path { - upsert_env( - env, - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, - socket_path, - ); - } -} - -fn provider_spiffe_socket_path<'a>(params: &'a SandboxPodParams<'a>) -> Option<&'a str> { - params - .provider_spiffe_enabled - .then_some(params.provider_spiffe_workload_api_socket_path) -} - -fn spiffe_socket_mount_path(socket_path: &str) -> String { - Path::new(socket_path) - .parent() - .and_then(Path::to_str) - .filter(|path| !path.is_empty() && *path != "/") - .expect("provider SPIFFE socket path should be validated before pod rendering") - .to_string() + env } fn upsert_env(env: &mut Vec, name: &str, value: &str) { @@ -4452,10 +6143,6 @@ fn remove_env(env: &mut Vec, name: &str) { env.retain(|item| item.get("name").and_then(|value| value.as_str()) != Some(name)); } -fn remove_volume_mount(volume_mounts: &mut Vec, name: &str) { - volume_mounts.retain(|mount| mount.get("name").and_then(|value| value.as_str()) != Some(name)); -} - /// Extract a string value from the template's `platform_config` Struct. fn platform_config_string(template: &SandboxTemplate, key: &str) -> Option { let config = template.platform_config.as_ref()?; @@ -4490,6 +6177,40 @@ fn platform_config_struct(template: &SandboxTemplate, key: &str) -> Option std::collections::HashMap { + object + .data + .pointer("/spec/podTemplate/spec/containers/0/env") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter_map(|entry| { + let name = entry.get("name")?.as_str()?; + let value = entry.get("value")?.as_str()?; + (!name.starts_with("OPENSHELL_")).then(|| (name.to_string(), value.to_string())) + }) + .collect() +} + +fn required_sandbox_annotation( + object: &DynamicObject, + name: &str, +) -> Result { + object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(name)) + .cloned() + .ok_or_else(|| { + KubernetesDriverError::Precondition(format!( + "Sandbox resource is missing required annotation {name}" + )) + }) +} + fn status_from_object(obj: &DynamicObject) -> Option { let status = obj.data.get("status")?; let status_obj = status.as_object()?; @@ -4532,6 +6253,117 @@ fn status_from_object(obj: &DynamicObject) -> Option { }) } +async fn create_or_validate_sandbox_runtime_fence( + policies: &Api, + expected: &NetworkPolicy, +) -> Result<(), KubernetesDriverError> { + let name = expected.metadata.name.as_deref().unwrap_or_default(); + match tokio::time::timeout(KUBE_API_TIMEOUT, policies.get_opt(name)).await { + Ok(Ok(Some(existing))) => { + return validate_sandbox_runtime_fence(&existing, expected); + } + Ok(Ok(None)) => {} + Ok(Err(error)) => return Err(KubernetesDriverError::from_kube(error)), + Err(_) => { + return Err(KubernetesDriverError::Message( + "timed out reading sandbox-runtime workload fence".to_string(), + )); + } + } + + match tokio::time::timeout( + KUBE_API_TIMEOUT, + policies.create(&PostParams::default(), expected), + ) + .await + { + Ok(Ok(_)) => Ok(()), + Ok(Err(KubeError::Api(error))) if error.code == 409 => { + let existing = tokio::time::timeout(KUBE_API_TIMEOUT, policies.get(name)) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out validating existing sandbox-runtime workload fence".to_string(), + ) + })? + .map_err(KubernetesDriverError::from_kube)?; + validate_sandbox_runtime_fence(&existing, expected) + } + Ok(Err(error)) => Err(KubernetesDriverError::from_kube(error)), + Err(_) => Err(KubernetesDriverError::Message( + "timed out creating sandbox-runtime workload fence".to_string(), + )), + } +} + +fn validate_sandbox_runtime_fence( + existing: &NetworkPolicy, + expected: &NetworkPolicy, +) -> Result<(), KubernetesDriverError> { + if sandbox_runtime_fence_matches(existing, expected) { + Ok(()) + } else { + let name = expected.metadata.name.as_deref().unwrap_or_default(); + Err(KubernetesDriverError::Precondition(format!( + "sandbox-runtime workload fence {name} exists but does not match the intended enforcement" + ))) + } +} + +fn sandbox_runtime_fence_matches(existing: &NetworkPolicy, expected: &NetworkPolicy) -> bool { + fn normalized_spec(mut spec: Option) -> Option { + if let Some(spec) = spec.as_mut() { + // The Kubernetes API server omits explicitly empty rule arrays when it + // persists a NetworkPolicy. For a policy type named in `policyTypes`, + // an omitted rule array and an empty rule array both deny all traffic. + if spec.egress.as_ref().is_some_and(Vec::is_empty) { + spec.egress = None; + } + if spec.ingress.as_ref().is_some_and(Vec::is_empty) { + spec.ingress = None; + } + } + spec + } + + fn contains_required_metadata( + actual: &Option>, + required: &Option>, + ) -> bool { + required.as_ref().is_none_or(|required| { + actual.as_ref().is_some_and(|actual| { + required + .iter() + .all(|(key, value)| actual.get(key) == Some(value)) + }) + }) + } + + normalized_spec(existing.spec.clone()) == normalized_spec(expected.spec.clone()) + && contains_required_metadata(&existing.metadata.labels, &expected.metadata.labels) + && contains_required_metadata( + &existing.metadata.annotations, + &expected.metadata.annotations, + ) +} + +fn sandbox_runtime_namespace_fence_generation_matches( + policy: &NetworkPolicy, + sandbox: &DynamicObject, +) -> bool { + let Some(annotations) = sandbox.metadata.annotations.as_ref() else { + return false; + }; + policy.metadata.uid.as_deref() + == annotations + .get(ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID) + .map(String::as_str) + && policy.metadata.resource_version.as_deref() + == annotations + .get(ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_VERSION) + .map(String::as_str) +} + fn kubernetes_sandbox_has_stopped_condition(obj: &DynamicObject) -> bool { obj.data .get("status") @@ -4638,18 +6470,68 @@ fn sandbox_operating_state_patch( running: bool, ) -> serde_json::Value { if api_version == SANDBOX_VERSION_V1BETA1 { - serde_json::json!({ - "metadata": {"resourceVersion": resource_version}, - "spec": {"operatingMode": if running { "Running" } else { "Suspended" }} - }) + if running { + serde_json::json!({ + "metadata": {"resourceVersion": resource_version}, + "spec": {"operatingMode": "Running"} + }) + } else { + sandbox_runtime_rollback_patch(api_version, resource_version) + } } else { - serde_json::json!({ - "metadata": {"resourceVersion": resource_version}, - "spec": {"replicas": i32::from(running)} - }) + if running { + serde_json::json!({ + "metadata": {"resourceVersion": resource_version}, + "spec": {"replicas": 1} + }) + } else { + sandbox_runtime_rollback_patch(api_version, resource_version) + } } } +fn sandbox_runtime_rollback_patch(api_version: &str, resource_version: &str) -> serde_json::Value { + let desired_state = if api_version == SANDBOX_VERSION_V1BETA1 { + serde_json::json!({"operatingMode": "Suspended"}) + } else { + serde_json::json!({"replicas": 0}) + }; + serde_json::json!({ + "metadata": { + "resourceVersion": resource_version, + "annotations": { + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING: "true", + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT: openshell_core::time::now_ms().to_string(), + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION: "stop", + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE: SandboxRuntimeBootstrapPhase::RollingBack.as_str(), + ANNOTATION_SANDBOX_RUNTIME_READINESS: "unavailable", + }, + }, + "spec": desired_state, + }) +} + +fn sandbox_runtime_rollback_completion_patch(resource_version: &str) -> serde_json::Value { + serde_json::json!({ + "metadata": { + "resourceVersion": resource_version, + "annotations": { + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_SESSION: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_GENERATION: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_VERSION: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_READINESS: "unavailable", + }, + } + }) +} + fn condition_from_value(value: &serde_json::Value) -> Option { let obj = value.as_object()?; Some(SandboxCondition { @@ -4689,9 +6571,9 @@ fn spawn_namespace_label_watcher( tokio::spawn(async move { let mut retry_attempt = 0; + let mut relisted_names = std::collections::BTreeSet::new(); loop { let mut stream = watcher::watcher(ns_api.clone(), watcher_config.clone()).boxed(); - let mut relisted_names = BTreeSet::new(); loop { let event = tokio::select! { @@ -4704,9 +6586,43 @@ fn spawn_namespace_label_watcher( } }; match event { - Ok(Some(event)) => { + Ok(Some(Event::Apply(ns))) => { + retry_attempt = 0; + if let Some(name) = ns.metadata.name.as_deref() + && allowlist.insert(name.to_string()) + { + info!(namespace = name, "operator namespace added to allowlist"); + } + } + Ok(Some(Event::Delete(ns))) => { + retry_attempt = 0; + if let Some(name) = ns.metadata.name.as_deref() + && allowlist.remove(name) + { + info!( + namespace = name, + "operator namespace removed from allowlist" + ); + } + } + Ok(Some(Event::Init)) => { retry_attempt = 0; - apply_namespace_watch_event(&allowlist, &mut relisted_names, event); + relisted_names.clear(); + } + Ok(Some(Event::InitApply(ns))) => { + retry_attempt = 0; + if let Some(name) = ns.metadata.name { + relisted_names.insert(name); + } + } + Ok(Some(Event::InitDone)) => { + retry_attempt = 0; + let count = relisted_names.len(); + allowlist.replace(std::mem::take(&mut relisted_names)); + info!( + total = count, + "operator namespace allowlist replaced from full relist" + ); } Ok(None) => { warn!("operator namespace watcher stream ended unexpectedly"); @@ -4748,7 +6664,7 @@ fn namespace_watcher_retry_delay(attempt: u32, jitter_seed: u64) -> Duration { Duration::from_secs(base_secs + jitter_secs) } -fn load_namespace_file(path: &Path) -> Result, String> { +fn load_namespace_file(path: &Path) -> Result, String> { let contents = std::fs::read_to_string(path) .map_err(|e| format!("failed to read {}: {e}", path.display()))?; let names: Vec = serde_json::from_str(&contents) @@ -4756,47 +6672,6 @@ fn load_namespace_file(path: &Path) -> Result, String> { Ok(names.into_iter().collect()) } -fn apply_namespace_watch_event( - allowlist: &OperatorNamespaceAllowlist, - relisted_names: &mut BTreeSet, - event: Event, -) { - match event { - Event::Apply(ns) => { - if let Some(name) = ns.metadata.name - && allowlist.insert(name.clone()) - { - info!(namespace = name, "operator namespace added to allowlist"); - } - } - Event::Delete(ns) => { - if let Some(name) = ns.metadata.name - && allowlist.remove(&name) - { - info!( - namespace = name, - "operator namespace removed from allowlist" - ); - } - } - Event::Init => relisted_names.clear(), - Event::InitApply(ns) => { - if let Some(name) = ns.metadata.name { - relisted_names.insert(name); - } - } - Event::InitDone => { - // Readers must see a complete snapshot, including during interrupted relists. - let count = relisted_names.len(); - allowlist.replace(std::mem::take(relisted_names)); - info!( - total = count, - "operator namespace allowlist replaced from full relist" - ); - } - } -} - fn spawn_namespace_file_watcher( path: PathBuf, allowlist: OperatorNamespaceAllowlist, @@ -4936,6 +6811,16 @@ mod tests { static ENV_LOCK: std::sync::LazyLock> = std::sync::LazyLock::new(|| std::sync::Mutex::new(())); + #[test] + fn boundary_authority_uses_stable_service_dns_name() { + let names = SandboxRuntimeNames::new("sandbox-1"); + + assert_eq!( + boundary_service_authority("workspace-a", &names, 5500), + "os-boundary-sandbox-1.workspace-a.svc:5500" + ); + } + #[tokio::test] async fn tracing_create_sandbox_failure_exports_a_kubernetes_operation_span() { use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; @@ -5034,6 +6919,32 @@ mod tests { }) } + #[test] + fn resource_version_conflicts_are_not_reported_as_duplicate_sandboxes() { + let conflict = KubeError::Api(kube::core::ErrorResponse { + status: "Failure".to_string(), + message: "the object has been modified".to_string(), + reason: "Conflict".to_string(), + code: 409, + }); + assert!(is_kube_resource_version_conflict(&conflict)); + assert!(matches!( + KubernetesDriverError::from_kube(conflict), + KubernetesDriverError::Message(_) + )); + + let duplicate = KubeError::Api(kube::core::ErrorResponse { + status: "Failure".to_string(), + message: "already exists".to_string(), + reason: "AlreadyExists".to_string(), + code: 409, + }); + assert!(matches!( + KubernetesDriverError::from_kube(duplicate), + KubernetesDriverError::AlreadyExists + )); + } + fn expired_watch_error() -> watcher::Error { watcher::Error::WatchError(kube::core::ErrorResponse { status: "Failure".to_string(), @@ -5044,7 +6955,7 @@ mod tests { } #[tokio::test] - async fn sandbox_watcher_error_does_not_hide_relist() { + async fn sandbox_watcher_error_does_not_hide_restarted_recovery_event() { let recovered = DynamicObject { types: None, metadata: ObjectMeta { @@ -5053,15 +6964,10 @@ mod tests { }, data: serde_json::json!({}), }; - let source = futures::stream::iter([ - Err(expired_watch_error()), - Ok(Event::Init), - Ok(Event::InitApply(recovered)), - Ok(Event::InitDone), - ]); + let source = + futures::stream::iter([Err(expired_watch_error()), Ok(Event::InitApply(recovered))]); let mut stream = continue_on_watcher_errors(source, "sandbox-resource"); - assert!(matches!(stream.next().await, Some(Event::Init))); let event = stream .next() .await @@ -5070,7 +6976,6 @@ mod tests { panic!("expected kube-runtime recovery to emit InitApply"); }; assert_eq!(object.metadata.name.as_deref(), Some("recovered-sandbox")); - assert!(matches!(stream.next().await, Some(Event::InitDone))); assert!( stream.next().await.is_none(), "source closure must be preserved" @@ -5097,15 +7002,17 @@ mod tests { }, data: serde_json::json!({}), }; - let source = futures::stream::iter([ - Err(expired_watch_error()), - Ok(Event::Init), - Ok(Event::InitApply(recovered)), - Ok(Event::InitDone), - ]) - .chain(futures::stream::pending()); + let source = + futures::stream::iter([Err(expired_watch_error()), Ok(Event::InitApply(recovered))]) + .chain(futures::stream::pending()); let sandbox_stream = recovering_watcher_stream(source, "sandbox-resource").boxed(); - let mut outward = cluster_wide_watch_stream(sandbox_stream, "default".to_string()); + let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig::default()); + let mut outward = cluster_wide_watch_stream( + sandbox_stream, + "default".to_string(), + driver.watch_client.clone(), + driver, + ); let event = outward .next() @@ -5128,76 +7035,24 @@ mod tests { } #[tokio::test] - async fn kubernetes_event_watcher_error_does_not_hide_relist() { + async fn kubernetes_event_watcher_error_does_not_hide_restarted_recovery_event() { let source = futures::stream::iter([ Err(expired_watch_error()), - Ok(Event::Init), Ok(Event::InitApply(KubeEventObj::default())), - Ok(Event::InitDone), ]); - let mut stream = continue_on_watcher_errors(source, "kubernetes-event"); - - let event = stream - .next() - .await - .expect("410 Expired must not terminate the watcher stream"); - assert!(matches!(event, Event::Init)); - assert!(matches!(stream.next().await, Some(Event::InitApply(_)))); - assert!(matches!(stream.next().await, Some(Event::InitDone))); - assert!( - stream.next().await.is_none(), - "source closure must be preserved" - ); - } - - #[test] - fn namespace_relist_replaces_only_completed_snapshots() { - let allowlist = OperatorNamespaceAllowlist::from_set(BTreeSet::from(["old".to_string()])); - let mut pending = BTreeSet::new(); - let namespace = |name: &str| Namespace { - metadata: ObjectMeta { - name: Some(name.to_string()), - ..Default::default() - }, - ..Default::default() - }; - let config = KubernetesComputeConfig { - workspace_mode: WorkspaceMode::Operator, - ..Default::default() - }; - - apply_namespace_watch_event(&allowlist, &mut pending, Event::Init); - apply_namespace_watch_event( - &allowlist, - &mut pending, - Event::InitApply(namespace("partial")), - ); - assert!(accepts_auth_namespace(&config, Some(&allowlist), "old")); - assert!(!accepts_auth_namespace( - &config, - Some(&allowlist), - "partial" - )); - - apply_namespace_watch_event(&allowlist, &mut pending, Event::Init); - apply_namespace_watch_event(&allowlist, &mut pending, Event::InitApply(namespace("new"))); - apply_namespace_watch_event(&allowlist, &mut pending, Event::InitDone); - assert!(accepts_auth_namespace(&config, Some(&allowlist), "new")); - assert!(!accepts_auth_namespace(&config, Some(&allowlist), "old")); - assert!(!accepts_auth_namespace( - &config, - Some(&allowlist), - "partial" - )); - - apply_namespace_watch_event(&allowlist, &mut pending, Event::Apply(namespace("live"))); - assert!(accepts_auth_namespace(&config, Some(&allowlist), "live")); - apply_namespace_watch_event(&allowlist, &mut pending, Event::Delete(namespace("live"))); - assert!(!accepts_auth_namespace(&config, Some(&allowlist), "live")); + let mut stream = continue_on_watcher_errors(source, "kubernetes-event"); - apply_namespace_watch_event(&allowlist, &mut pending, Event::Init); - apply_namespace_watch_event(&allowlist, &mut pending, Event::InitDone); - assert!(!accepts_auth_namespace(&config, Some(&allowlist), "new")); + let event = stream + .next() + .await + .expect("410 Expired must not terminate the watcher stream"); + let Event::InitApply(_event) = event else { + panic!("expected kube-runtime recovery to emit InitApply"); + }; + assert!( + stream.next().await.is_none(), + "source closure must be preserved" + ); } fn authenticated_token_review(username: &str) -> TokenReviewStatus { @@ -5323,7 +7178,7 @@ mod tests { } #[test] - fn pod_identity_requires_matching_uid_annotation_and_controlling_owner() { + fn pod_identity_requires_matching_uid_annotation_and_expected_owner_kind() { let owner = sandbox_owner_for_test("sandbox-a", "sandbox-uid-a"); let pod = Pod { metadata: ObjectMeta { @@ -5340,7 +7195,7 @@ mod tests { validate_pod_uid(&pod, "pod-uid-a").expect("matching pod UID"); assert_eq!(pod_sandbox_id(&pod).unwrap(), "sandbox-id-a"); - assert_eq!(sandbox_owner_reference(&pod).unwrap(), &owner); + assert_eq!(sandbox_owner_reference(&pod, true).unwrap(), &owner); let error = validate_pod_uid(&pod, "other-pod-uid").unwrap_err(); assert_eq!(error.code(), tonic::Code::PermissionDenied); @@ -5352,8 +7207,133 @@ mod tests { let mut non_controlling = pod; non_controlling.metadata.owner_references.as_mut().unwrap()[0].controller = Some(false); - let error = sandbox_owner_reference(&non_controlling).unwrap_err(); + let error = sandbox_owner_reference(&non_controlling, true).unwrap_err(); assert_eq!(error.code(), tonic::Code::PermissionDenied); + assert!(sandbox_owner_reference(&non_controlling, false).is_ok()); + } + + #[test] + fn proxy_control_identity_requires_exact_pair_and_role_labels() { + let sandbox_id = "sandbox-id-a"; + let mut pod = Pod { + metadata: ObjectMeta { + labels: Some(BTreeMap::from([ + (LABEL_SANDBOX_ID.to_string(), sandbox_id.to_string()), + ( + BOUNDARY_PAIR_LABEL.to_string(), + crate::sandbox_runtime::pair_label_value(sandbox_id), + ), + (BOUNDARY_ROLE_LABEL.to_string(), "supervisor".to_string()), + ])), + ..Default::default() + }, + ..Default::default() + }; + validate_proxy_control_labels(&pod, sandbox_id).unwrap(); + + pod.metadata + .labels + .as_mut() + .unwrap() + .insert(BOUNDARY_ROLE_LABEL.to_string(), "workload".to_string()); + assert_eq!( + validate_proxy_control_labels(&pod, sandbox_id) + .unwrap_err() + .code(), + tonic::Code::PermissionDenied + ); + } + + #[test] + fn sandbox_authentication_requires_the_paired_supervisor() { + require_proxy_control_authentication(true).expect("paired supervisor is trusted"); + assert_eq!( + require_proxy_control_authentication(false) + .expect_err("workload JWT must not authenticate directly") + .code(), + tonic::Code::PermissionDenied + ); + } + + #[test] + fn sandbox_runtime_fence_validation_accepts_api_normalization_and_injected_metadata() { + let names = SandboxRuntimeNames::new("sandbox-id-a"); + let mut expected = workload_fence("namespace-a", &names, 5000).workload_policy; + expected.metadata.labels = Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])); + + let mut persisted = expected.clone(); + persisted.spec.as_mut().unwrap().egress = None; + persisted + .metadata + .labels + .as_mut() + .unwrap() + .insert("admission.example/injected".to_string(), "true".to_string()); + assert!(sandbox_runtime_fence_matches(&persisted, &expected)); + assert!(validate_sandbox_runtime_fence(&persisted, &expected).is_ok()); + + persisted.spec.as_mut().unwrap().policy_types = Some(vec!["Ingress".to_string()]); + assert!(!sandbox_runtime_fence_matches(&persisted, &expected)); + assert!(matches!( + validate_sandbox_runtime_fence(&persisted, &expected), + Err(KubernetesDriverError::Precondition(_)) + )); + } + + #[test] + fn sandbox_runtime_bootstrap_marker_and_age_gate_rollback() { + let started = Duration::from_hours(490_896); + let mut object: DynamicObject = serde_json::from_value(serde_json::json!({ + "apiVersion": "agents.x-k8s.io/v1beta1", + "kind": "Sandbox", + "metadata": { + "name": "sandbox-a", + "creationTimestamp": "2020-01-01T00:00:00Z", + "annotations": { + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING: "true", + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT: started.as_millis().to_string(), + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION: "restart", + } + } + })) + .unwrap(); + assert!(sandbox_runtime_bootstrap_in_progress(&object)); + assert_eq!( + sandbox_runtime_bootstrap_operation(&object), + Some("restart") + ); + let started_at = SystemTime::UNIX_EPOCH + started; + assert!(!sandbox_runtime_bootstrap_is_stale( + &object, + started_at + SANDBOX_RUNTIME_BOOTSTRAP_GRACE - Duration::from_secs(1), + SANDBOX_RUNTIME_BOOTSTRAP_GRACE + )); + assert!(sandbox_runtime_bootstrap_is_stale( + &object, + started_at + SANDBOX_RUNTIME_BOOTSTRAP_GRACE, + SANDBOX_RUNTIME_BOOTSTRAP_GRACE + )); + object + .metadata + .annotations + .as_mut() + .unwrap() + .remove(ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT); + assert!(sandbox_runtime_bootstrap_is_stale( + &object, + started_at, + SANDBOX_RUNTIME_BOOTSTRAP_GRACE + )); + object + .metadata + .annotations + .as_mut() + .unwrap() + .remove(ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING); + assert!(!sandbox_runtime_bootstrap_in_progress(&object)); } #[test] @@ -5387,11 +7367,20 @@ mod tests { fn lifecycle_patch_uses_version_specific_operating_state() { let beta_stop = sandbox_operating_state_patch(SANDBOX_VERSION_V1BETA1, "42", false); assert_eq!(beta_stop["metadata"]["resourceVersion"], "42"); + assert_eq!( + beta_stop["metadata"]["annotations"][ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING], + "true" + ); + assert_eq!( + beta_stop["metadata"]["annotations"][ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE], + SandboxRuntimeBootstrapPhase::RollingBack.as_str() + ); assert_eq!(beta_stop["spec"]["operatingMode"], "Suspended"); assert!(beta_stop["spec"].get("replicas").is_none()); let alpha_start = sandbox_operating_state_patch(SANDBOX_VERSION_V1ALPHA1, "43", true); assert_eq!(alpha_start["metadata"]["resourceVersion"], "43"); + assert!(alpha_start["metadata"].get("annotations").is_none()); assert_eq!(alpha_start["spec"]["replicas"], 1); assert!(alpha_start["spec"].get("operatingMode").is_none()); } @@ -5593,7 +7582,7 @@ mod tests { ..Default::default() }; - let err = kubernetes_driver_config_for_spec(sandbox.spec.as_ref(), None).unwrap_err(); + let err = kubernetes_driver_config_for_spec(sandbox.spec.as_ref()).unwrap_err(); assert!(err.contains("unknown field")); assert!(err.contains("gpu_device_ids")); } @@ -5942,7 +7931,7 @@ mod tests { ..SandboxSpec::default() }; - let err = kubernetes_driver_config_for_spec(Some(&spec), None).unwrap_err(); + let err = kubernetes_driver_config_for_spec(Some(&spec)).unwrap_err(); assert!(err.contains("/var/run/secrets/openshell")); } @@ -5970,7 +7959,7 @@ mod tests { ..SandboxSpec::default() }; - kubernetes_driver_config_for_spec(Some(&spec), None) + kubernetes_driver_config_for_spec(Some(&spec)) .expect("SPIFFE workload path should only be protected when SPIFFE is enabled"); } @@ -6085,7 +8074,11 @@ mod tests { CLIENT_TLS_VOLUME_NAME, SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, SPIFFE_WORKLOAD_API_VOLUME_NAME, - SUPERVISOR_VOLUME_NAME, + SANDBOX_RUNTIME_VOLUME_NAME, + SANDBOX_STATE_VOLUME_NAME, + SANDBOX_BOOTSTRAP_VOLUME_NAME, + SANDBOX_POD_IDENTITY_VOLUME_NAME, + SANDBOX_PROXY_CA_VOLUME_NAME, WORKSPACE_VOLUME_NAME, ] { let template = SandboxTemplate { @@ -6108,12 +8101,7 @@ mod tests { #[test] fn reserved_kubernetes_volume_names_cover_managed_pod_volumes() { - let params = SandboxPodParams { - client_tls_secret_name: "openshell-client-tls-secret", - provider_spiffe_enabled: true, - provider_spiffe_workload_api_socket_path: "/spiffe-workload-api/spire-agent.sock", - ..SandboxPodParams::default() - }; + let params = SandboxPodParams::default(); let pod_template = sandbox_template_to_k8s( &SandboxTemplate::default(), false, @@ -6136,36 +8124,6 @@ mod tests { } } - #[test] - fn driver_config_rejects_runtime_provider_spiffe_mount_path() { - let spec = SandboxSpec { - template: Some(SandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "volumes": [{ - "name": "user-data", - "persistent_volume_claim": {"claim_name": "pvc-user-data"} - }], - "containers": { - "agent": { - "volume_mounts": [{ - "name": "user-data", - "mount_path": "/custom-spiffe" - }] - } - } - }))), - ..SandboxTemplate::default() - }), - ..SandboxSpec::default() - }; - - let err = - kubernetes_driver_config_for_spec(Some(&spec), Some("/custom-spiffe/spire-agent.sock")) - .unwrap_err(); - - assert!(err.contains("/custom-spiffe")); - } - #[test] fn validate_rejects_zero_gpu_count() { let sandbox = Sandbox { @@ -6218,757 +8176,161 @@ mod tests { ); assert_eq!( - metadata.get(PROGRESS_COMPLETE_STEP_KEY).map(String::as_str), - Some(PROGRESS_STEP_PULLING_IMAGE) - ); - assert_eq!( - metadata - .get(PROGRESS_COMPLETE_LABEL_KEY) - .map(String::as_str), - Some("Image pulled (42 MB)") - ); - assert_eq!( - metadata.get(PROGRESS_ACTIVE_STEP_KEY).map(String::as_str), - Some(PROGRESS_STEP_STARTING_SANDBOX) - ); - } - - #[test] - fn supervisor_sideload_injects_run_as_user_zero() { - let mut pod_template = serde_json::json!({ - "spec": { - "containers": [{ - "name": "agent", - "image": "custom-image:latest", - "securityContext": { - "capabilities": { - "add": ["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE", "SYSLOG"] - } - } - }] - } - }); - - apply_supervisor_sideload( - &mut pod_template, - "custom-image:latest", - Some("IfNotPresent"), - SupervisorSideloadMethod::InitContainer, - 1500, // sandbox_uid - 1500, // sandbox_gid - ); - - let sc = &pod_template["spec"]["containers"][0]["securityContext"]; - assert_eq!(sc["runAsUser"], 0, "runAsUser must be 0 for supervisor"); - // Capabilities should be preserved - assert!( - sc["capabilities"]["add"] - .as_array() - .unwrap() - .contains(&serde_json::json!("SYS_ADMIN")) - ); - } - - #[test] - fn supervisor_sideload_replaces_spoofed_identity_environment() { - let mut pod_template = serde_json::json!({ - "spec": { - "containers": [{ - "name": "agent", - "image": "custom-image:latest", - "env": [ - {"name": openshell_core::sandbox_env::OCI_IMAGE_USER, "value": "spoofed"}, - {"name": openshell_core::sandbox_env::SANDBOX_UID, "value": "9999"}, - {"name": openshell_core::sandbox_env::SANDBOX_GID, "value": "9999"}, - {"name": openshell_core::sandbox_env::OCI_IMAGE_USER, "value": "duplicate"} - ] - }] - } - }); - - apply_supervisor_sideload( - &mut pod_template, - "supervisor-image:latest", - Some("IfNotPresent"), - SupervisorSideloadMethod::InitContainer, - 1500, - 1600, - ); - - let agent = &pod_template["spec"]["containers"][0]; - let env = agent["env"].as_array().unwrap(); - for name in [ - openshell_core::sandbox_env::OCI_IMAGE_USER, - openshell_core::sandbox_env::SANDBOX_UID, - openshell_core::sandbox_env::SANDBOX_GID, - ] { - assert_eq!( - env.iter().filter(|item| item["name"] == name).count(), - 1, - "{name} must have one driver-owned value" - ); - } - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::OCI_IMAGE_USER), - Some("") - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::SANDBOX_UID), - Some("1500") - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::SANDBOX_GID), - Some("1600") - ); - } - - #[test] - fn supervisor_sideload_adds_security_context_when_missing() { - let mut pod_template = serde_json::json!({ - "spec": { - "containers": [{ - "name": "agent", - "image": "custom-image:latest" - }] - } - }); - - apply_supervisor_sideload( - &mut pod_template, - "supervisor-image:latest", - Some("IfNotPresent"), - SupervisorSideloadMethod::InitContainer, - 1000, // sandbox_uid - 1000, // sandbox_gid - ); - - let sc = &pod_template["spec"]["containers"][0]["securityContext"]; - assert_eq!( - sc["runAsUser"], 0, - "runAsUser must be 0 even when no prior securityContext" - ); - } - - #[test] - fn supervisor_sideload_injects_emptydir_volume_init_container_and_mount() { - let mut pod_template = serde_json::json!({ - "spec": { - "containers": [{ - "name": "agent", - "image": "custom-image:latest" - }] - } - }); - - apply_supervisor_sideload( - &mut pod_template, - "supervisor-image:latest", - Some("IfNotPresent"), - SupervisorSideloadMethod::InitContainer, - 1000, // sandbox_uid - 1000, // sandbox_gid - ); - - // Volume should be an emptyDir - let volumes = pod_template["spec"]["volumes"] - .as_array() - .expect("volumes should exist"); - assert_eq!(volumes.len(), 1); - assert_eq!(volumes[0]["name"], SUPERVISOR_VOLUME_NAME); - assert!( - volumes[0]["emptyDir"].is_object(), - "volume should be emptyDir, not hostPath" - ); - - // Init container should use the supervisor image, not the sandbox image - let init_containers = pod_template["spec"]["initContainers"] - .as_array() - .expect("initContainers should exist"); - assert_eq!(init_containers.len(), 1); - assert_eq!(init_containers[0]["name"], SUPERVISOR_INIT_CONTAINER_NAME); - assert_eq!(init_containers[0]["image"], "supervisor-image:latest"); - assert_eq!(init_containers[0]["imagePullPolicy"], "IfNotPresent"); - - // The init container must invoke the binary directly with - // `copy-self ` rather than depending on shell utilities. - let init_command = init_containers[0]["command"] - .as_array() - .expect("init container command should be set"); - assert_eq!(init_command.len(), 3, "expected [binary, copy-self, dest]"); - assert_eq!(init_command[0], SUPERVISOR_IMAGE_BINARY_PATH); - assert_eq!(init_command[1], "copy-self"); - assert_eq!( - init_command[2].as_str().unwrap(), - format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox") - ); - assert!( - !init_command.iter().any(|v| v == "sh"), - "init container must not depend on a shell" - ); - - // `--workdir` is optional for standalone supervisor invocations and - // has no implicit default, so Kubernetes must pass its fixed workspace. - let command = pod_template["spec"]["containers"][0]["command"] - .as_array() - .expect("command should be set"); - assert_eq!( - command[0].as_str().unwrap(), - format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox") - ); - assert_eq!( - command, - serde_json::json!([ - format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"), - "--workdir", - driver_mounts::DEFAULT_WORKSPACE_ROOT - ]) - .as_array() - .unwrap() - ); - - // Agent volume mount should be read-only - let mounts = pod_template["spec"]["containers"][0]["volumeMounts"] - .as_array() - .expect("volumeMounts should exist"); - assert_eq!(mounts.len(), 1); - assert_eq!(mounts[0]["name"], SUPERVISOR_VOLUME_NAME); - assert_eq!(mounts[0]["mountPath"], SUPERVISOR_MOUNT_PATH); - assert_eq!(mounts[0]["readOnly"], true); - } - - #[test] - fn supervisor_sideload_image_volume_injects_image_source_without_init_container() { - let mut pod_template = serde_json::json!({ - "spec": { - "containers": [{ - "name": "agent", - "image": "custom-image:latest" - }] - } - }); - - apply_supervisor_sideload( - &mut pod_template, - "supervisor-image:latest", - Some("IfNotPresent"), - SupervisorSideloadMethod::ImageVolume, - 1000, // sandbox_uid - 1000, // sandbox_gid - ); - - let volumes = pod_template["spec"]["volumes"] - .as_array() - .expect("volumes should exist"); - assert_eq!(volumes.len(), 1); - assert_eq!(volumes[0]["name"], SUPERVISOR_VOLUME_NAME); - assert_eq!(volumes[0]["image"]["reference"], "supervisor-image:latest"); - assert_eq!(volumes[0]["image"]["pullPolicy"], "IfNotPresent"); - assert!( - volumes[0]["emptyDir"].is_null(), - "image volume method must not use emptyDir" - ); - - assert!( - pod_template["spec"]["initContainers"].is_null(), - "image volume method must not inject init containers" - ); - - let command = pod_template["spec"]["containers"][0]["command"] - .as_array() - .expect("command should be set"); - assert_eq!( - command[0].as_str().unwrap(), - format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox") - ); - - let sc = &pod_template["spec"]["containers"][0]["securityContext"]; - assert_eq!(sc["runAsUser"], 0); - - let mounts = pod_template["spec"]["containers"][0]["volumeMounts"] - .as_array() - .expect("volumeMounts should exist"); - assert_eq!(mounts[0]["name"], SUPERVISOR_VOLUME_NAME); - assert_eq!(mounts[0]["mountPath"], SUPERVISOR_MOUNT_PATH); - assert_eq!(mounts[0]["readOnly"], true); - } - - #[test] - fn supervisor_image_volume_omits_pull_policy_when_unspecified() { - let mut pod_template = serde_json::json!({ - "spec": { - "containers": [{ - "name": "agent", - "image": "custom-image:latest" - }] - } - }); - - apply_supervisor_sideload( - &mut pod_template, - "supervisor-image:latest", - None, - SupervisorSideloadMethod::ImageVolume, - 1000, // sandbox_uid - 1000, // sandbox_gid + metadata.get(PROGRESS_COMPLETE_STEP_KEY).map(String::as_str), + Some(PROGRESS_STEP_PULLING_IMAGE) ); - - let volume = &pod_template["spec"]["volumes"][0]; - assert_eq!(volume["image"]["reference"], "supervisor-image:latest"); - assert!( - volume["image"].get("pullPolicy").is_none(), - "pullPolicy should be omitted when unspecified" + assert_eq!( + metadata + .get(PROGRESS_COMPLETE_LABEL_KEY) + .map(String::as_str), + Some("Image pulled (42 MB)") + ); + assert_eq!( + metadata.get(PROGRESS_ACTIVE_STEP_KEY).map(String::as_str), + Some(PROGRESS_STEP_STARTING_SANDBOX) ); } #[test] - fn sidecar_topology_renders_process_agent_and_network_sidecar() { + fn sandbox_runtime_renders_credential_free_boundary_workload() { let params = SandboxPodParams { - topology: SupervisorTopology::Sidecar, - supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, - supervisor_image: "supervisor-image:latest", - supervisor_image_pull_policy: Some("IfNotPresent"), - grpc_endpoint: "https://openshell-gateway.openshell.svc:8080", - client_tls_secret_name: "openshell-client-tls", - proxy_uid: 2200, + default_image: "agent:latest", + image_pull_policy: Some(crate::KubernetesImagePullPolicy::Always), + sandbox_runtime_image: "sandbox-runtime-image:latest", + sandbox_runtime_image_pull_policy: Some(crate::KubernetesImagePullPolicy::Never), + sandbox_id: "sandbox-123", sandbox_uid: 1500, sandbox_gid: 1500, ..SandboxPodParams::default() }; let pod_template = sandbox_template_to_k8s( - &SandboxTemplate { - image: "agent-image:latest".to_string(), - environment: std::collections::HashMap::from([ - ( - openshell_core::sandbox_env::OCI_IMAGE_USER.to_string(), - "spoofed".to_string(), - ), - ( - openshell_core::sandbox_env::SANDBOX_UID.to_string(), - "9999".to_string(), - ), - ( - openshell_core::sandbox_env::SANDBOX_GID.to_string(), - "9999".to_string(), - ), - ]), - ..SandboxTemplate::default() - }, + &SandboxTemplate::default(), false, &std::collections::HashMap::new(), - false, + true, ¶ms, ); + let agent = &pod_template["spec"]["containers"][0]; + assert_eq!(agent["imagePullPolicy"], "Always"); - assert_eq!(pod_template["spec"]["shareProcessNamespace"], true); - assert_eq!(pod_template["spec"]["securityContext"]["fsGroup"], 1500); - let containers = pod_template["spec"]["containers"].as_array().unwrap(); - assert_eq!(containers.len(), 2); - - let agent = containers - .iter() - .find(|container| container["name"] == "agent") - .unwrap(); assert_eq!( agent["command"], serde_json::json!([ - format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"), - "--mode=process", - "--workdir", - driver_mounts::DEFAULT_WORKSPACE_ROOT + format!("{SANDBOX_RUNTIME_MOUNT_PATH}/openshell-sandbox"), + "--bootstrap", + BOUNDARY_CONFIG_PATH ]) ); assert_eq!(agent["securityContext"]["runAsUser"], 1500); assert_eq!(agent["securityContext"]["runAsGroup"], 1500); assert_eq!(agent["securityContext"]["runAsNonRoot"], true); - assert_eq!(agent["securityContext"]["allowPrivilegeEscalation"], false); assert_eq!( agent["securityContext"]["capabilities"], - serde_json::json!({ - "drop": ["ALL"] - }) - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::ENDPOINT), - None - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME), - None - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::TLS_CA), - None - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::K8S_SA_TOKEN_FILE), - None - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::SSH_SOCKET_PATH), - Some(SIDECAR_SSH_SOCKET_FILE) - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET), - Some(SIDECAR_CONTROL_SOCKET) - ); - assert_eq!(rendered_env(agent, "OPENSHELL_SUPERVISOR_READY_FILE"), None); - assert_eq!(rendered_env(agent, "OPENSHELL_ENTRYPOINT_PID_FILE"), None); - assert_eq!( - rendered_env(agent, "OPENSHELL_SIDECAR_POLICY_SNAPSHOT_FILE"), - None - ); - assert_eq!( - rendered_env(agent, "OPENSHELL_SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE"), - None - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::PROXY_TLS_DIR), - Some(SIDECAR_TLS_MOUNT_PATH) - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::SANDBOX_UID), - Some("1500") - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::OCI_IMAGE_USER), - Some("") - ); - - let sidecar = containers - .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) - .unwrap(); - assert_eq!(sidecar["image"], "supervisor-image:latest"); - assert_eq!(sidecar["imagePullPolicy"], "IfNotPresent"); - assert_eq!( - sidecar["command"], - serde_json::json!([SUPERVISOR_IMAGE_BINARY_PATH, "--mode=network"]) - ); - assert_eq!(sidecar["securityContext"]["runAsUser"], 0); - assert_eq!(sidecar["securityContext"]["runAsGroup"], 1500); - assert_eq!(sidecar["securityContext"]["runAsNonRoot"], false); - assert_eq!( - sidecar["securityContext"]["allowPrivilegeEscalation"], - false - ); - assert_eq!( - sidecar["securityContext"]["capabilities"], - serde_json::json!({ - "drop": ["ALL"], - "add": ["SYS_PTRACE", "DAC_READ_SEARCH"] - }) - ); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::ENDPOINT), - Some("https://openshell-gateway.openshell.svc:8080") - ); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::SSH_SOCKET_PATH), - Some(SIDECAR_SSH_SOCKET_FILE) - ); - assert!( - SIDECAR_SSH_SOCKET_FILE.starts_with('@'), - "sidecar SSH relay must use a Linux abstract socket" - ); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::SANDBOX_UID), - Some("1500") - ); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::SANDBOX_GID), - Some("1500") - ); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::OCI_IMAGE_USER), - Some("") - ); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET), - Some(SIDECAR_CONTROL_SOCKET) - ); - assert_eq!( - rendered_env(sidecar, "OPENSHELL_SIDECAR_POLICY_SNAPSHOT_FILE"), - None - ); - assert_eq!( - rendered_env(sidecar, "OPENSHELL_SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE"), - None + serde_json::json!({"drop": ["ALL"]}) ); + assert!(agent["securityContext"]["capabilities"]["add"].is_null()); + assert_eq!(pod_template["spec"]["securityContext"]["fsGroup"], 1500); assert_eq!( - rendered_env( - sidecar, - openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY - ), - None + pod_template["spec"]["securityContext"]["seccompProfile"]["type"], + "RuntimeDefault" ); - assert_eq!(rendered_env(sidecar, "OPENSHELL_ENTRYPOINT_PID_FILE"), None); + assert_eq!(pod_template["spec"]["dnsPolicy"], "None"); assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::PROXY_TLS_DIR), - Some(SIDECAR_TLS_MOUNT_PATH) + pod_template["spec"]["dnsConfig"]["nameservers"], + serde_json::json!(["127.0.0.53"]) ); assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::TLS_CA), - Some("/etc/openshell-tls/proxy/client/ca.crt") + pod_template["spec"]["schedulingGates"], + serde_json::json!([{"name": SANDBOX_BOOTSTRAP_SCHEDULING_GATE}]) ); - let sidecar_mounts = sidecar["volumeMounts"].as_array().unwrap(); - assert!( - !sidecar_mounts - .iter() - .any(|mount| mount["name"] == "openshell-client-tls"), - "runtime sidecar should use the init-copied TLS files, not the root-owned Secret mount" - ); - let agent_mounts = agent["volumeMounts"].as_array().unwrap(); - assert!( - !agent_mounts - .iter() - .any(|mount| mount["name"] == "openshell-sa-token"), - "agent container must not mount gateway bootstrap token in sidecar topology" - ); - assert!( - !agent_mounts - .iter() - .any(|mount| mount["name"] == "openshell-client-tls"), - "agent container must not mount gateway client TLS secret in sidecar topology" - ); - let volumes = pod_template["spec"]["volumes"].as_array().unwrap(); - let sa_token = volumes - .iter() - .find(|volume| volume["name"] == "openshell-sa-token") - .unwrap(); - assert_eq!(sa_token["projected"]["defaultMode"], 0o440); - let client_tls = volumes + let sandbox_bootstrap = pod_template["spec"]["initContainers"] + .as_array() + .unwrap() .iter() - .find(|volume| volume["name"] == "openshell-client-tls") + .find(|container| container["name"] == "openshell-sandbox-bootstrap") .unwrap(); - assert_eq!(client_tls["secret"]["defaultMode"], 0o440); - - let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); - let network_init = init_containers + assert_eq!(sandbox_bootstrap["image"], "sandbox-runtime-image:latest"); + assert_eq!(sandbox_bootstrap["imagePullPolicy"], "Never"); + let workspace_init = pod_template["spec"]["initContainers"] + .as_array() + .unwrap() .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) + .find(|container| container["name"] == WORKSPACE_INIT_CONTAINER_NAME) .unwrap(); - assert_eq!(network_init["image"], "supervisor-image:latest"); - assert_eq!(network_init["imagePullPolicy"], "IfNotPresent"); assert_eq!( - network_init["command"], + workspace_init["command"], serde_json::json!([ - SUPERVISOR_IMAGE_BINARY_PATH, - "--mode=network-init", - "--proxy-uid", - "0", - "--proxy-gid", - "1500", - "--sidecar-state-dir", - SIDECAR_STATE_MOUNT_PATH, - "--sidecar-tls-dir", - SIDECAR_TLS_MOUNT_PATH + format!("{SANDBOX_RUNTIME_MOUNT_PATH}/openshell-sandbox"), + "seed-workspace" ]) ); + assert_eq!(workspace_init["securityContext"]["runAsUser"], 1500); + assert_eq!(workspace_init["imagePullPolicy"], "Always"); assert_eq!( - network_init["securityContext"]["capabilities"], - serde_json::json!({ - "drop": ["ALL"], - "add": ["NET_ADMIN", "NET_RAW", "CHOWN", "FOWNER"] - }) - ); - let network_init_mounts = network_init["volumeMounts"].as_array().unwrap(); - assert!(network_init_mounts.iter().any(|mount| { - mount["name"] == "openshell-client-tls" - && mount["mountPath"] == "/etc/openshell-tls/client" - })); - } - - #[test] - fn sidecar_topology_can_relax_process_binary_aware_network_policy() { - let params = SandboxPodParams { - topology: SupervisorTopology::Sidecar, - supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, - supervisor_image: "supervisor-image:latest", - proxy_uid: 2200, - sandbox_uid: 1500, - sandbox_gid: 1500, - process_binary_aware_network_policy: false, - ..SandboxPodParams::default() - }; - let pod_template = sandbox_template_to_k8s( - &SandboxTemplate { - image: "agent-image:latest".to_string(), - ..SandboxTemplate::default() - }, - false, - &std::collections::HashMap::new(), - false, - ¶ms, + workspace_init["securityContext"]["capabilities"], + serde_json::json!({"drop": ["ALL"]}) ); - - let containers = pod_template["spec"]["containers"].as_array().unwrap(); - let sidecar = containers - .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) - .unwrap(); - assert_eq!(sidecar["securityContext"]["runAsUser"], 2200); - assert_eq!(sidecar["securityContext"]["runAsGroup"], 1500); - assert_eq!(sidecar["securityContext"]["runAsNonRoot"], true); assert_eq!( - sidecar["securityContext"]["allowPrivilegeEscalation"], - false + rendered_env(agent, openshell_core::sandbox_env::ENDPOINT), + None ); assert_eq!( - sidecar["securityContext"]["capabilities"], - serde_json::json!({ - "drop": ["ALL"] - }) + rendered_env(agent, openshell_core::sandbox_env::K8S_SA_TOKEN_FILE), + None ); assert_eq!( - rendered_env( - sidecar, - openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY - ), - Some("relaxed") + pod_template["metadata"]["labels"][BOUNDARY_ROLE_LABEL], + "workload" ); - let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); - let network_init = init_containers - .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) - .unwrap(); - assert_eq!(network_init["command"][3], "2200"); - } - #[test] - fn sidecar_topology_adds_shared_state_and_tls_volumes() { - let params = SandboxPodParams { - topology: SupervisorTopology::Sidecar, - supervisor_sideload_method: SupervisorSideloadMethod::ImageVolume, - supervisor_image: "supervisor-image:latest", - grpc_endpoint: "http://openshell-gateway.openshell.svc:8080", - ..SandboxPodParams::default() - }; - let pod_template = sandbox_template_to_k8s( - &SandboxTemplate::default(), - false, - &std::collections::HashMap::new(), - false, - ¶ms, + let mounts = agent["volumeMounts"].as_array().unwrap(); + assert!(mounts.iter().any(|mount| { + mount["name"] == SANDBOX_RUNTIME_VOLUME_NAME && mount["readOnly"] == true + })); + assert!(mounts.iter().any(|mount| { + mount["name"] == SANDBOX_STATE_VOLUME_NAME && mount["readOnly"].is_null() + })); + assert!(mounts.iter().any(|mount| { + mount["name"] == SANDBOX_POD_IDENTITY_VOLUME_NAME + && mount["mountPath"] == SANDBOX_POD_IDENTITY_MOUNT_PATH + && mount["readOnly"] == true + })); + assert!(mounts.iter().any(|mount| { + mount["name"] == SANDBOX_PROXY_CA_VOLUME_NAME + && mount["mountPath"] == SANDBOX_PROXY_CA_MOUNT_PATH + && mount["readOnly"].is_null() + })); + assert!( + !mounts + .iter() + .any(|mount| mount["name"] == SANDBOX_BOOTSTRAP_VOLUME_NAME) ); - - let volumes = pod_template["spec"]["volumes"].as_array().unwrap(); assert!( - volumes + !mounts .iter() - .any(|volume| volume["name"] == SIDECAR_STATE_VOLUME_NAME) + .any(|mount| mount["name"] == CLIENT_TLS_VOLUME_NAME) ); assert!( - volumes + !mounts .iter() - .any(|volume| volume["name"] == SIDECAR_TLS_VOLUME_NAME) + .any(|mount| mount["name"] == SERVICE_ACCOUNT_TOKEN_VOLUME_NAME) ); - assert!(volumes.iter().any(|volume| { - volume["name"] == SUPERVISOR_VOLUME_NAME && volume["image"].is_object() - })); - let containers = pod_template["spec"]["containers"].as_array().unwrap(); - let sidecar = containers + let volumes = pod_template["spec"]["volumes"].as_array().unwrap(); + let pod_identity = volumes .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) + .find(|volume| volume["name"] == SANDBOX_POD_IDENTITY_VOLUME_NAME) .unwrap(); assert_eq!( - sidecar["securityContext"]["capabilities"], - serde_json::json!({ - "drop": ["ALL"], - "add": ["SYS_PTRACE", "DAC_READ_SEARCH"] - }) - ); - assert_eq!(sidecar["securityContext"]["runAsUser"], 0); - assert_eq!(sidecar["securityContext"]["runAsGroup"], 1000); - assert_eq!(sidecar["securityContext"]["runAsNonRoot"], false); - assert_eq!( - sidecar["securityContext"]["allowPrivilegeEscalation"], - false + pod_identity["downwardAPI"]["items"], + serde_json::json!([{"path": "uid", "fieldRef": {"fieldPath": "metadata.uid"}}]) ); - - for container_name in ["agent", SUPERVISOR_NETWORK_SIDECAR_NAME] { - let container = containers - .iter() - .find(|container| container["name"] == container_name) - .unwrap(); - let mounts = container["volumeMounts"].as_array().unwrap(); - assert!(mounts.iter().any(|mount| { - mount["name"] == SIDECAR_STATE_VOLUME_NAME - && mount["mountPath"] == SIDECAR_STATE_MOUNT_PATH - })); - assert!(mounts.iter().any(|mount| { - mount["name"] == SIDECAR_TLS_VOLUME_NAME - && mount["mountPath"] == SIDECAR_TLS_MOUNT_PATH - })); - } - let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); - let network_init = init_containers - .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) - .unwrap(); - assert_eq!(network_init["command"][3], "0"); - } - - #[test] - fn sidecar_topology_rejects_proxy_uid_matching_sandbox_uid() { - let params = SandboxPodParams { - topology: SupervisorTopology::Sidecar, - proxy_uid: 1500, - sandbox_uid: 1500, - ..SandboxPodParams::default() - }; - - let err = validate_sidecar_proxy_identity(¶ms).unwrap_err(); - assert!(matches!(err, KubernetesDriverError::Precondition(_))); - assert!(err.to_string().contains("proxy_uid")); } /// Regression test: TLS mount path must match env var paths. /// The volume is mounted at a specific path and the env vars must point to /// files within that same path, otherwise the sandbox will fail to start /// with "No such file or directory" errors. - #[test] - fn tls_env_vars_match_volume_mount_path() { - // The mount path used in pod template construction - const TLS_MOUNT_PATH: &str = "/etc/openshell-tls/client"; - - // Build env with TLS enabled - let mut env = Vec::new(); - apply_required_env( - &mut env, - "sandbox-1", - "my-sandbox", - "https://endpoint:8080", - "0.0.0.0:2222", - true, // tls_enabled - None, - ); - - // Extract the TLS-related env vars - let get_env = |name: &str| -> Option { - env.iter() - .find(|e| e.get("name").and_then(|v| v.as_str()) == Some(name)) - .and_then(|e| e.get("value").and_then(|v| v.as_str()).map(String::from)) - }; - - let tls_ca = get_env("OPENSHELL_TLS_CA").expect("OPENSHELL_TLS_CA must be set"); - let tls_cert = get_env("OPENSHELL_TLS_CERT").expect("OPENSHELL_TLS_CERT must be set"); - let tls_key = get_env("OPENSHELL_TLS_KEY").expect("OPENSHELL_TLS_KEY must be set"); - - // All TLS paths must be within the mount path - assert!( - tls_ca.starts_with(TLS_MOUNT_PATH), - "OPENSHELL_TLS_CA path '{tls_ca}' must start with mount path '{TLS_MOUNT_PATH}'" - ); - assert!( - tls_cert.starts_with(TLS_MOUNT_PATH), - "OPENSHELL_TLS_CERT path '{tls_cert}' must start with mount path '{TLS_MOUNT_PATH}'" - ); - assert!( - tls_key.starts_with(TLS_MOUNT_PATH), - "OPENSHELL_TLS_KEY path '{tls_key}' must start with mount path '{TLS_MOUNT_PATH}'" - ); - } #[test] fn gpu_sandbox_adds_runtime_class_and_gpu_limit() { @@ -7303,87 +8665,9 @@ mod tests { let resources = &pod_template["spec"]["containers"][0]["resources"]; assert_eq!(resources["limits"]["cpu"], serde_json::json!("500m")); - assert_eq!(resources["limits"]["memory"], serde_json::json!("2Gi")); - assert_eq!(resources["requests"]["cpu"], serde_json::json!("500m")); - assert_eq!(resources["requests"]["memory"], serde_json::json!("2Gi")); - } - - #[test] - fn host_aliases_injected_when_gateway_ip_set() { - let pod_template = { - let params = SandboxPodParams { - host_gateway_ip: "172.17.0.1", - ..Default::default() - }; - sandbox_template_to_k8s( - &SandboxTemplate::default(), - false, - &std::collections::HashMap::new(), - true, - ¶ms, - ) - }; - - let host_aliases = pod_template["spec"]["hostAliases"] - .as_array() - .expect("hostAliases should exist"); - assert_eq!(host_aliases.len(), 1); - assert_eq!(host_aliases[0]["ip"], "172.17.0.1"); - let hostnames = host_aliases[0]["hostnames"] - .as_array() - .expect("hostnames should exist"); - assert!(hostnames.contains(&serde_json::json!("host.docker.internal"))); - assert!(hostnames.contains(&serde_json::json!("host.openshell.internal"))); - } - - #[test] - fn host_aliases_not_injected_when_gateway_ip_empty() { - let pod_template = { - let params = SandboxPodParams::default(); - sandbox_template_to_k8s( - &SandboxTemplate::default(), - false, - &std::collections::HashMap::new(), - true, - ¶ms, - ) - }; - - assert!( - pod_template["spec"]["hostAliases"].is_null(), - "hostAliases should not be present when host_gateway_ip is empty" - ); - } - - #[test] - fn tls_secret_volume_uses_restrictive_default_mode() { - let template = SandboxTemplate::default(); - let pod_template = { - let params = SandboxPodParams { - client_tls_secret_name: "my-tls-secret", - ..Default::default() - }; - sandbox_template_to_k8s( - &template, - false, - &std::collections::HashMap::new(), - true, - ¶ms, - ) - }; - - let volumes = pod_template["spec"]["volumes"] - .as_array() - .expect("volumes should exist"); - let tls_vol = volumes - .iter() - .find(|v| v["name"] == CLIENT_TLS_VOLUME_NAME) - .expect("TLS volume should exist"); - assert_eq!( - tls_vol["secret"]["defaultMode"], - 256, // 0o400 - "TLS secret volume must use mode 0400 to prevent sandbox user from reading the private key" - ); + assert_eq!(resources["limits"]["memory"], serde_json::json!("2Gi")); + assert_eq!(resources["requests"]["cpu"], serde_json::json!("500m")); + assert_eq!(resources["requests"]["memory"], serde_json::json!("2Gi")); } // ----------------------------------------------------------------------- @@ -7404,8 +8688,9 @@ mod tests { apply_workspace_persistence( &mut pod_template, "openshell/sandbox:latest", - Some("IfNotPresent"), - 1000, // sandbox_gid + Some(crate::KubernetesImagePullPolicy::IfNotPresent), + Some(1000), // sandbox_gid + None, ); // Init container @@ -7463,8 +8748,9 @@ mod tests { apply_workspace_persistence( &mut pod_template, "my-custom-image:v2", - Some("IfNotPresent"), - 1000, + Some(crate::KubernetesImagePullPolicy::IfNotPresent), + Some(1000), + None, ); let init_image = pod_template["spec"]["initContainers"][0]["image"] @@ -7487,7 +8773,13 @@ mod tests { } }); - apply_workspace_persistence(&mut pod_template, "img:latest", Some("Always"), 1000); + apply_workspace_persistence( + &mut pod_template, + "img:latest", + Some(crate::KubernetesImagePullPolicy::Always), + Some(1000), + None, + ); let cmd = pod_template["spec"]["initContainers"][0]["command"] .as_array() @@ -7515,10 +8807,7 @@ mod tests { #[test] fn workspace_persistence_skipped_when_inject_workspace_false() { - let params = SandboxPodParams { - supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, - ..SandboxPodParams::default() - }; + let params = SandboxPodParams::default(); let pod_template = sandbox_template_to_k8s( &SandboxTemplate::default(), false, @@ -7566,65 +8855,6 @@ mod tests { ) } - #[test] - fn app_armor_profile_omitted_by_default() { - let pod_template = default_template_to_k8s(false); - assert!( - pod_template["spec"]["containers"][0]["securityContext"]["appArmorProfile"].is_null(), - "appArmorProfile must be omitted when no profile is configured" - ); - } - - #[test] - fn app_armor_profile_renders_unconfined() { - let profile = AppArmorProfile::Unconfined; - let params = SandboxPodParams { - app_armor_profile: Some(&profile), - ..Default::default() - }; - let pod_template = sandbox_template_to_k8s( - &SandboxTemplate::default(), - false, - &std::collections::HashMap::new(), - true, - ¶ms, - ); - - assert_eq!( - pod_template["spec"]["containers"][0]["securityContext"]["appArmorProfile"], - serde_json::json!({ "type": "Unconfined" }) - ); - assert_eq!( - pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]["add"][0], - serde_json::json!("SYS_ADMIN"), - "AppArmor rendering must preserve required capabilities" - ); - } - - #[test] - fn app_armor_profile_renders_localhost_profile() { - let profile = AppArmorProfile::Localhost("openshell-supervisor".to_string()); - let params = SandboxPodParams { - app_armor_profile: Some(&profile), - ..Default::default() - }; - let pod_template = sandbox_template_to_k8s( - &SandboxTemplate::default(), - false, - &std::collections::HashMap::new(), - true, - ¶ms, - ); - - assert_eq!( - pod_template["spec"]["containers"][0]["securityContext"]["appArmorProfile"], - serde_json::json!({ - "type": "Localhost", - "localhostProfile": "openshell-supervisor" - }) - ); - } - #[test] fn user_namespaces_disabled_by_default() { let pod_template = default_template_to_k8s(false); @@ -7632,11 +8862,10 @@ mod tests { pod_template["spec"]["hostUsers"].is_null(), "hostUsers must not be set when user namespaces are disabled" ); - let caps = pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]["add"] - .as_array() - .unwrap(); - assert_eq!(caps.len(), 4); - assert!(!caps.contains(&serde_json::json!("SETUID"))); + let capabilities = + &pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]; + assert!(capabilities["add"].is_null()); + assert_eq!(capabilities["drop"], serde_json::json!(["ALL"])); } #[test] @@ -7650,19 +8879,12 @@ mod tests { } #[test] - fn user_namespaces_adds_extra_capabilities() { + fn user_namespaces_preserve_capability_free_posture() { let pod_template = default_template_to_k8s(true); - let caps = pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]["add"] - .as_array() - .unwrap(); - assert!(caps.contains(&serde_json::json!("SYS_ADMIN"))); - assert!(caps.contains(&serde_json::json!("NET_ADMIN"))); - assert!(caps.contains(&serde_json::json!("SYS_PTRACE"))); - assert!(caps.contains(&serde_json::json!("SYSLOG"))); - assert!(caps.contains(&serde_json::json!("SETUID"))); - assert!(caps.contains(&serde_json::json!("SETGID"))); - assert!(caps.contains(&serde_json::json!("DAC_READ_SEARCH"))); - assert_eq!(caps.len(), 7); + let capabilities = + &pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]; + assert!(capabilities["add"].is_null()); + assert_eq!(capabilities["drop"], serde_json::json!(["ALL"])); } #[test] @@ -7686,10 +8908,10 @@ mod tests { serde_json::json!(false), "per-sandbox user namespace intent must set hostUsers: false" ); - let caps = pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]["add"] - .as_array() - .unwrap(); - assert!(caps.contains(&serde_json::json!("SETUID"))); + let capabilities = + &pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]; + assert!(capabilities["add"].is_null()); + assert_eq!(capabilities["drop"], serde_json::json!(["ALL"])); } #[test] @@ -7715,14 +8937,10 @@ mod tests { pod_template["spec"]["hostUsers"].is_null(), "per-sandbox user namespace intent must override the cluster default" ); - let caps = pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]["add"] - .as_array() - .unwrap(); - assert_eq!( - caps.len(), - 4, - "extra capabilities must not be added when user namespaces are disabled" - ); + let capabilities = + &pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]; + assert!(capabilities["add"].is_null()); + assert_eq!(capabilities["drop"], serde_json::json!(["ALL"])); } #[test] @@ -7894,51 +9112,6 @@ mod tests { ); } - #[test] - fn provider_spiffe_mounts_csi_socket_and_keeps_sa_token_bootstrap() { - let params = SandboxPodParams { - sandbox_id: "sandbox-123", - sandbox_name: "sandbox", - provider_spiffe_enabled: true, - provider_spiffe_workload_api_socket_path: "/spiffe-workload-api/spire-agent.sock", - ..SandboxPodParams::default() - }; - let pod_template = sandbox_template_to_k8s( - &SandboxTemplate::default(), - false, - &std::collections::HashMap::new(), - true, - ¶ms, - ); - - let env = pod_template["spec"]["containers"][0]["env"] - .as_array() - .expect("env"); - assert!(env.iter().any(|e| { - e["name"] == openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET - && e["value"] == "/spiffe-workload-api/spire-agent.sock" - })); - assert!(env.iter().any(|e| { - e["name"] == openshell_core::sandbox_env::K8S_SA_TOKEN_FILE - && e["value"] == "/var/run/secrets/openshell/token" - })); - - let volumes = pod_template["spec"]["volumes"].as_array().expect("volumes"); - assert!(volumes.iter().any(|volume| { - volume["name"] == SPIFFE_WORKLOAD_API_VOLUME_NAME - && volume["csi"]["driver"] == "csi.spiffe.io" - })); - assert!(volumes.iter().any(|volume| { - volume["name"] == SERVICE_ACCOUNT_TOKEN_VOLUME_NAME - && volume["projected"]["sources"][0]["serviceAccountToken"]["path"] == "token" - })); - - assert_eq!( - pod_template["metadata"]["labels"][LABEL_MANAGED_BY], - serde_json::json!(LABEL_MANAGED_BY_VALUE) - ); - } - #[test] fn log_level_propagates_as_env_var_to_sandbox_pod() { let spec = SandboxSpec { @@ -7988,7 +9161,7 @@ mod tests { } #[test] - fn sandbox_pod_clears_unsupported_network_capabilities() { + fn sandbox_pod_drops_legacy_network_capability_environment() { let spec = SandboxSpec { environment: std::collections::HashMap::from([( openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), @@ -8000,14 +9173,9 @@ mod tests { let env = cr["spec"]["podTemplate"]["spec"]["containers"][0]["env"] .as_array() .unwrap(); - let entries = env - .iter() - .filter(|entry| { - entry["name"] == openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES - }) - .collect::>(); - assert_eq!(entries.len(), 1); - assert_eq!(entries[0]["value"], serde_json::json!("")); + assert!(!env.iter().any(|entry| { + entry["name"] == openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES + })); } #[test] @@ -8456,103 +9624,6 @@ mod tests { ); } - #[test] - fn upstream_proxy_is_injected_only_into_network_supervisors() { - let params = SandboxPodParams { - topology: SupervisorTopology::Sidecar, - supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, - supervisor_image: "supervisor-image:latest", - https_proxy: Some("http://proxy.corp.example:8080"), - no_proxy: Some(".svc.cluster.local,10.96.0.0/12"), - proxy_auth_secret_name: Some("corporate-proxy-auth"), - proxy_auth_secret_key: Some("credentials"), - proxy_auth_allow_insecure: true, - proxy_connect_by_hostname: true, - sandbox_uid: 1500, - sandbox_gid: 1500, - ..SandboxPodParams::default() - }; - let pod = sandbox_template_to_k8s( - &SandboxTemplate::default(), - false, - &std::collections::HashMap::new(), - false, - ¶ms, - ); - let containers = pod["spec"]["containers"].as_array().unwrap(); - let network = containers - .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) - .unwrap(); - let command = network["command"].as_array().unwrap(); - assert!(command.iter().any(|arg| arg == "--upstream-proxy")); - assert!(command.iter().any(|arg| arg == "--upstream-no-proxy")); - let auth_file_index = command - .iter() - .position(|arg| arg == "--upstream-proxy-auth-file") - .unwrap(); - assert_eq!( - command[auth_file_index + 1], - openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH - ); - assert!( - command - .iter() - .any(|arg| arg == "--upstream-proxy-auth-allow-insecure") - ); - assert!( - command - .iter() - .any(|arg| arg == "--upstream-proxy-connect-by-hostname") - ); - assert!( - network["volumeMounts"] - .as_array() - .unwrap() - .iter() - .any(|mount| mount["name"] == UPSTREAM_PROXY_AUTH_VOLUME_NAME) - ); - - let init = pod["spec"]["initContainers"] - .as_array() - .unwrap() - .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) - .unwrap(); - assert!(!init["command"].as_array().unwrap().iter().any(|arg| { - arg.as_str() - .is_some_and(|arg| arg.starts_with("--upstream-")) - })); - let agent = containers - .iter() - .find(|container| container["name"] == "agent") - .unwrap(); - assert!( - !agent["volumeMounts"] - .as_array() - .unwrap() - .iter() - .any(|mount| mount["name"] == UPSTREAM_PROXY_AUTH_VOLUME_NAME) - ); - assert!(!agent["env"].as_array().unwrap().iter().any(|entry| { - entry["value"] == "corporate-proxy-auth" || entry["value"] == "credentials" - })); - - let volume = pod["spec"]["volumes"] - .as_array() - .unwrap() - .iter() - .find(|volume| volume["name"] == UPSTREAM_PROXY_AUTH_VOLUME_NAME) - .unwrap(); - assert_eq!(volume["secret"]["secretName"], "corporate-proxy-auth"); - assert_eq!(volume["secret"]["items"][0]["key"], "credentials"); - assert_eq!( - volume["secret"]["items"][0]["path"], - upstream_proxy_auth_file_name() - ); - assert_eq!(volume["secret"]["defaultMode"], 0o440); - } - #[test] fn sandbox_lookup_selector_always_includes_gateway_id() { let sel = sandbox_lookup_selector_for("sb-123", "gw-42"); @@ -8783,4 +9854,240 @@ mod tests { assert!(gpu.default_selection_supported); assert!(gpu.count_selection_supported); } + + #[test] + fn sandbox_runtime_control_availability_requires_a_ready_pod() { + let mut pod = Pod::default(); + assert_eq!( + sandbox_runtime_control_availability_from_pod(&pod), + SandboxRuntimeControlAvailability::Unavailable + ); + pod.status = Some( + serde_json::from_value(serde_json::json!({ + "conditions": [{"type": "Ready", "status": "True"}] + })) + .expect("valid Pod status"), + ); + assert_eq!( + sandbox_runtime_control_availability_from_pod(&pod), + SandboxRuntimeControlAvailability::Available + ); + } + + #[test] + fn sandbox_runtime_readiness_transitions_bump_the_watched_cr() { + let unavailable = sandbox_runtime_readiness_transition_patch("42", "unavailable"); + let ready = sandbox_runtime_readiness_transition_patch("42", "ready"); + + assert_eq!(unavailable["metadata"]["resourceVersion"], "42"); + assert_eq!(ready["metadata"]["resourceVersion"], "42"); + assert_eq!( + unavailable["metadata"]["annotations"][ANNOTATION_SANDBOX_RUNTIME_READINESS], + "unavailable" + ); + assert_eq!( + ready["metadata"]["annotations"][ANNOTATION_SANDBOX_RUNTIME_READINESS], + "ready" + ); + assert_ne!(unavailable, ready); + } + + #[test] + fn sandbox_runtime_bootstrap_completion_is_resource_version_guarded_and_publishes_ready() { + let patch = sandbox_runtime_bootstrap_completion_patch("42"); + + assert_eq!(patch["metadata"]["resourceVersion"], "42"); + assert_eq!( + patch["metadata"]["annotations"][ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING], + serde_json::Value::Null + ); + assert_eq!( + patch["metadata"]["annotations"][ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT], + serde_json::Value::Null + ); + assert_eq!( + patch["metadata"]["annotations"][ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION], + serde_json::Value::Null + ); + assert_eq!( + patch["metadata"]["annotations"][ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE], + serde_json::Value::Null + ); + assert_eq!( + patch["metadata"]["annotations"][ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_SESSION], + serde_json::Value::Null + ); + assert_eq!( + patch["metadata"]["annotations"][ANNOTATION_SANDBOX_RUNTIME_READINESS], + "ready" + ); + } + + #[test] + fn sandbox_runtime_rollback_is_durable_until_cleanup_completes() { + let rollback = sandbox_runtime_rollback_patch(SANDBOX_VERSION_V1BETA1, "42"); + assert_eq!(rollback["metadata"]["resourceVersion"], "42"); + assert_eq!( + rollback["metadata"]["annotations"][ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING], + "true" + ); + assert_eq!( + rollback["metadata"]["annotations"][ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION], + "stop" + ); + assert_eq!( + rollback["metadata"]["annotations"][ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE], + SandboxRuntimeBootstrapPhase::RollingBack.as_str() + ); + assert_eq!(rollback["spec"]["operatingMode"], "Suspended"); + + let complete = sandbox_runtime_rollback_completion_patch("43"); + assert_eq!(complete["metadata"]["resourceVersion"], "43"); + for annotation in [ + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING, + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT, + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION, + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE, + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_SESSION, + ANNOTATION_SANDBOX_RUNTIME_GENERATION, + ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID, + ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID, + ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID, + ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_VERSION, + ] { + assert_eq!( + complete["metadata"]["annotations"][annotation], + serde_json::Value::Null + ); + } + } + + #[test] + fn sandbox_runtime_bootstrap_completion_waits_for_runtime_ready() { + let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( + SANDBOX_GROUP, + SANDBOX_VERSION_V1BETA1, + SANDBOX_KIND, + )); + let mut sandbox = DynamicObject::new("sandbox", &resource); + sandbox.metadata.generation = Some(7); + sandbox.data = serde_json::json!({ + "status": { + "conditions": [{"type": "Suspended", "status": "True", "observedGeneration": 7}] + } + }); + assert!(!sandbox_runtime_runtime_is_ready(&sandbox)); + + sandbox.data["status"]["conditions"] = serde_json::json!([ + {"type": "Suspended", "status": "False", "observedGeneration": 6}, + {"type": "Ready", "status": "True", "observedGeneration": 6} + ]); + assert!(!sandbox_runtime_runtime_is_ready(&sandbox)); + + sandbox.data["status"]["conditions"] = serde_json::json!([ + {"type": "Suspended", "status": "False", "observedGeneration": 7}, + {"type": "Ready", "status": "True", "observedGeneration": 7} + ]); + assert!(sandbox_runtime_runtime_is_ready(&sandbox)); + + sandbox.data["status"]["conditions"] = serde_json::json!([ + {"type": "Suspended", "status": "True", "observedGeneration": 7}, + {"type": "Ready", "status": "True", "observedGeneration": 7} + ]); + assert!(!sandbox_runtime_runtime_is_ready(&sandbox)); + } + + #[test] + fn sandbox_runtime_readiness_is_downgraded_with_a_transient_reason() { + let mut sandbox = Sandbox { + status: Some(SandboxStatus { + conditions: vec![SandboxCondition { + r#type: "Ready".to_string(), + status: "True".to_string(), + ..Default::default() + }], + ..Default::default() + }), + ..Default::default() + }; + mark_sandbox_runtime_control_unavailable(&mut sandbox); + let ready = &sandbox.status.unwrap().conditions[0]; + assert_eq!(ready.status, "False"); + assert_eq!(ready.reason, "DependenciesNotReady"); + } + + #[test] + fn sandbox_runtime_bootstrap_does_not_publish_a_terminal_suspension() { + let mut sandbox = Sandbox { + status: Some(SandboxStatus { + conditions: vec![SandboxCondition { + r#type: SANDBOX_SUSPENDED_CONDITION.to_string(), + status: "True".to_string(), + reason: "PodTerminated".to_string(), + ..Default::default() + }], + ..Default::default() + }), + ..Default::default() + }; + + mark_sandbox_runtime_bootstrapping(&mut sandbox); + + let conditions = &sandbox.status.unwrap().conditions; + assert!( + !conditions + .iter() + .any(|condition| condition.r#type == SANDBOX_SUSPENDED_CONDITION) + ); + assert!(conditions.iter().any(|condition| { + condition.r#type == "Ready" + && condition.status == "False" + && condition.reason == "DependenciesNotReady" + })); + assert!(conditions.iter().any(|condition| { + condition.r#type == "Bootstrapping" + && condition.status == "True" + && condition.reason == "SandboxRuntimeGenerationStarting" + })); + } + + #[test] + fn completed_sandbox_runtime_bootstrap_preserves_real_suspension() { + let sandbox = Sandbox { + status: Some(SandboxStatus { + conditions: vec![SandboxCondition { + r#type: SANDBOX_SUSPENDED_CONDITION.to_string(), + status: "True".to_string(), + reason: "PodTerminated".to_string(), + ..Default::default() + }], + ..Default::default() + }), + ..Default::default() + }; + + assert!(sandbox.status.unwrap().conditions.iter().any(|condition| { + condition.r#type == SANDBOX_SUSPENDED_CONDITION && condition.status == "True" + })); + } + + #[test] + fn sandbox_runtime_should_run_tracks_both_sandbox_apis() { + let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( + SANDBOX_GROUP, + SANDBOX_VERSION_V1BETA1, + SANDBOX_KIND, + )); + let mut beta = DynamicObject::new("beta", &resource); + beta.data = serde_json::json!({"spec": {"operatingMode": "Suspended"}}); + assert!(!sandbox_runtime_should_run(&beta)); + beta.data = serde_json::json!({"spec": {"operatingMode": "Running"}}); + assert!(sandbox_runtime_should_run(&beta)); + + let mut alpha = DynamicObject::new("alpha", &resource); + alpha.data = serde_json::json!({"spec": {"replicas": 0}}); + assert!(!sandbox_runtime_should_run(&alpha)); + alpha.data = serde_json::json!({"spec": {"replicas": 1}}); + assert!(sandbox_runtime_should_run(&alpha)); + } } diff --git a/crates/openshell-driver-kubernetes/src/grpc.rs b/crates/openshell-driver-kubernetes/src/grpc.rs index 095752d842..82d54ecea0 100644 --- a/crates/openshell-driver-kubernetes/src/grpc.rs +++ b/crates/openshell-driver-kubernetes/src/grpc.rs @@ -158,19 +158,21 @@ impl ComputeDriver for ComputeDriverService { &self, request: Request, ) -> Result, Status> { - self.rpc_tracer - .trace(openshell_otel::rpc::CREATE_SANDBOX, async { - let sandbox = request - .into_inner() - .sandbox - .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; - self.driver - .create_sandbox(&sandbox) - .await - .map_err(|e| Status::from(openshell_core::ComputeDriverError::from(e)))?; - Ok(Response::new(CreateSandboxResponse {})) - }) - .await + Box::pin( + self.rpc_tracer + .trace(openshell_otel::rpc::CREATE_SANDBOX, async { + let sandbox = request + .into_inner() + .sandbox + .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; + self.driver + .create_sandbox(&sandbox) + .await + .map_err(|e| Status::from(openshell_core::ComputeDriverError::from(e)))?; + Ok(Response::new(CreateSandboxResponse {})) + }), + ) + .await } async fn stop_sandbox( @@ -198,21 +200,26 @@ impl ComputeDriver for ComputeDriverService { &self, request: Request, ) -> Result, Status> { - self.rpc_tracer - .trace(openshell_otel::rpc::START_SANDBOX, async { - let request = request.into_inner(); - if request.sandbox_id.is_empty() { - return Err(Status::invalid_argument("sandbox_id is required")); - } - self.driver - .start_sandbox(&request.sandbox_id) + Box::pin( + self.rpc_tracer + .trace(openshell_otel::rpc::START_SANDBOX, async { + let request = request.into_inner(); + if request.sandbox_id.is_empty() { + return Err(Status::invalid_argument("sandbox_id is required")); + } + Box::pin(self.driver.start_sandbox( + &request.sandbox_id, + &request.generation_id, + &request.launch_authentication, + )) .await .map_err(|error| { Status::from(openshell_core::ComputeDriverError::from(error)) })?; - Ok(Response::new(StartSandboxResponse {})) - }) - .await + Ok(Response::new(StartSandboxResponse {})) + }), + ) + .await } async fn delete_sandbox( @@ -250,10 +257,12 @@ impl ComputeDriver for ComputeDriverService { let stream = stream.map(|item| item.map_err(|err| Status::internal(err.to_string()))); Ok::(Box::pin(stream)) }; - self.rpc_tracer - .trace_stream(openshell_otel::rpc::WATCH_SANDBOXES, create_stream) - .await - .map(Response::new) + Box::pin( + self.rpc_tracer + .trace_stream(openshell_otel::rpc::WATCH_SANDBOXES, create_stream), + ) + .await + .map(Response::new) } async fn ensure_workspace( diff --git a/crates/openshell-driver-kubernetes/src/isolation.rs b/crates/openshell-driver-kubernetes/src/isolation.rs new file mode 100644 index 0000000000..af4a712589 --- /dev/null +++ b/crates/openshell-driver-kubernetes/src/isolation.rs @@ -0,0 +1,444 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Kubernetes provisioning for the shared authenticated boundary protocol. +//! +//! This module deliberately contains no lifecycle, process, network, identity, +//! or wire implementation. The driver places the sandbox runtime, binds +//! immutable Kubernetes resource identities, and provisions TCP coordinates; +//! `openshell-isolation-interface` and `openshell-sandbox` provide the common +//! control and boundary behavior. + +use std::collections::{BTreeMap, HashMap}; +use std::net::{IpAddr, SocketAddr}; +use std::path::PathBuf; + +use k8s_openapi::api::networking::v1::{ + NetworkPolicy, NetworkPolicyEgressRule, NetworkPolicyIngressRule, NetworkPolicyPeer, + NetworkPolicyPort, NetworkPolicySpec, +}; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; +use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; +use kube::core::ObjectMeta; +use openshell_isolation_interface::contract::{DriverFenceEvidence, ResolvedWorkloadIdentity}; +use openshell_sandbox_backend::boundary_protocol::{ + BoundaryConfig, BoundaryListener, GatewayVerificationKey, SandboxRuntimeDescriptor, + SandboxTlsClientConfig, SandboxTlsServerConfig, SandboxTransport, +}; + +/// Isolation backend implemented by the `OpenShell` sandbox runtime. +pub const BACKEND_NAME: &str = openshell_sandbox_backend::BACKEND_NAME; + +/// Label that binds the workload and supervisor pods in one unique pair. +pub const BOUNDARY_PAIR_LABEL: &str = "openshell.ai/boundary-pair"; + +/// Label distinguishing the two pods in a sandbox generation. +pub const BOUNDARY_ROLE_LABEL: &str = "openshell.ai/boundary-role"; + +const WORKLOAD_ROLE: &str = "workload"; +const SUPERVISOR_ROLE: &str = "supervisor"; + +/// Driver-owned inputs for the workload pod's Kubernetes network fence. +/// +/// This is the first phase of sandbox-runtime provisioning. The driver applies the +/// returned labels to the respective pods and creates the returned policy. It +/// then observes the policy UID and resourceVersion and supplies both to +/// `KubernetesSandboxRuntimeBoundarySpec`. +pub struct KubernetesSandboxRuntimeNetworkFenceSpec { + pub namespace: String, + pub policy_name: String, + pub supervisor_policy_name: String, + pub boundary_port: u16, +} + +/// Labels and policy needed to remove direct workload-pod egress. +pub struct KubernetesSandboxRuntimeNetworkFence { + pub workload_labels: BTreeMap, + pub control_labels: BTreeMap, + pub workload_policy: NetworkPolicy, + pub supervisor_policy: NetworkPolicy, +} + +impl KubernetesSandboxRuntimeNetworkFenceSpec { + /// Render the namespace-wide workload fence. + /// + /// Kubernetes `NetworkPolicy` is connection-aware: traffic returning over + /// the control-initiated boundary connection is allowed even though the + /// workload pod has no egress rules. The control pod remains responsible + /// for opening policy-approved upstream connections. + #[must_use] + pub fn provision(self) -> KubernetesSandboxRuntimeNetworkFence { + let workload_labels = role_labels(WORKLOAD_ROLE); + let control_labels = role_labels(SUPERVISOR_ROLE); + + let workload_policy = NetworkPolicy { + metadata: ObjectMeta { + name: Some(self.policy_name), + namespace: Some(self.namespace.clone()), + ..Default::default() + }, + spec: Some(NetworkPolicySpec { + pod_selector: LabelSelector { + match_labels: Some(workload_labels.clone()), + ..Default::default() + }, + policy_types: Some(vec!["Ingress".to_string(), "Egress".to_string()]), + // Any trusted OpenShell supervisor in this namespace may + // reach a sandbox listener. The Sandbox Protocol enforces the + // exact sandbox, generation, and Pod UID binding. + ingress: Some(vec![NetworkPolicyIngressRule { + from: Some(vec![NetworkPolicyPeer { + pod_selector: Some(LabelSelector { + match_labels: Some(control_labels.clone()), + ..Default::default() + }), + ..Default::default() + }]), + ports: Some(vec![NetworkPolicyPort { + port: Some(IntOrString::Int(i32::from(self.boundary_port))), + protocol: Some("TCP".to_string()), + ..Default::default() + }]), + }]), + // An explicit empty list selects the pod for egress and allows + // no new workload-initiated connections, including DNS and the + // Kubernetes API. Reply traffic for allowed ingress remains + // permitted by conforming NetworkPolicy implementations. + egress: Some(Vec::new()), + }), + }; + + // Namespace-wide default-deny policies are additive with this rule. + // Select only OpenShell supervisor pods and explicitly allow their + // policy-approved DNS and upstream connections. + let supervisor_policy = NetworkPolicy { + metadata: ObjectMeta { + name: Some(self.supervisor_policy_name), + namespace: Some(self.namespace), + ..Default::default() + }, + spec: Some(NetworkPolicySpec { + pod_selector: LabelSelector { + match_labels: Some(control_labels.clone()), + ..Default::default() + }, + policy_types: Some(vec!["Egress".to_string()]), + egress: Some(vec![NetworkPolicyEgressRule::default()]), + ..Default::default() + }), + }; + + KubernetesSandboxRuntimeNetworkFence { + workload_labels, + control_labels, + workload_policy, + supervisor_policy, + } + } +} + +fn role_labels(role: &str) -> BTreeMap { + BTreeMap::from([(BOUNDARY_ROLE_LABEL.to_string(), role.to_string())]) +} + +/// Driver-owned inputs that bind one workload/supervisor pair to one boundary. +/// +/// The driver constructs this only after Kubernetes has assigned every UID and +/// after it has observed the namespace workload-policy resource version. The +/// workload stays held until the matching boundary config and supervisor +/// resources have been installed. +pub struct KubernetesSandboxRuntimeBoundarySpec { + pub boundary_id: String, + pub generation: String, + pub session_id: openshell_core::SandboxSessionId, + pub session_rotation: openshell_core::jwt::SessionRotation, + pub gateway_id: String, + pub verification_keys: Vec, + pub namespace_uid: String, + pub sandbox_resource_uid: String, + pub workload_pod_uid: String, + pub workload_pod_uid_path: PathBuf, + pub supervisor_pod_uid: String, + pub egress_policy_uid: String, + pub egress_policy_resource_version: String, + pub boundary_listener: SocketAddr, + pub control_authority: String, + pub control_address: SocketAddr, + pub sandbox_tls: SandboxTlsServerConfig, + pub supervisor_tls: SandboxTlsClientConfig, + pub host_gateway_ip: Option, + pub workload_identity: ResolvedWorkloadIdentity, + pub child_env: HashMap, +} + +/// Protected workload-pod config and matching sandbox-runtime descriptor. +pub struct KubernetesSandboxRuntimeBoundaryProvisioning { + pub boundary_config: BoundaryConfig, + pub runtime_descriptor: SandboxRuntimeDescriptor, +} + +impl KubernetesSandboxRuntimeBoundarySpec { + /// Produce both sides of the common protocol from one observed Kubernetes + /// resource set so a stale or recreated object cannot be attached. + #[must_use] + pub fn provision(self) -> KubernetesSandboxRuntimeBoundaryProvisioning { + let resource_claims = BTreeMap::from([ + ("kubernetes.namespace_uid".to_string(), self.namespace_uid), + ( + "kubernetes.sandbox_resource_uid".to_string(), + self.sandbox_resource_uid, + ), + ( + "kubernetes.workload_pod_uid".to_string(), + self.workload_pod_uid, + ), + ( + "kubernetes.supervisor_pod_uid".to_string(), + self.supervisor_pod_uid, + ), + ( + "kubernetes.egress_policy_uid".to_string(), + self.egress_policy_uid, + ), + ( + "kubernetes.egress_policy_resource_version".to_string(), + self.egress_policy_resource_version, + ), + ]); + let driver_fence = DriverFenceEvidence::Kubernetes { + network_policy_uid: resource_claims["kubernetes.egress_policy_uid"].clone(), + network_policy_resource_version: + resource_claims["kubernetes.egress_policy_resource_version"].clone(), + ingress_isolated: true, + egress_isolated: true, + egress_rule_count: 0, + }; + KubernetesSandboxRuntimeBoundaryProvisioning { + boundary_config: BoundaryConfig { + boundary_id: self.boundary_id.clone(), + generation: self.generation.clone(), + session_id: self.session_id, + session_rotation: self.session_rotation, + gateway_id: self.gateway_id, + verification_keys: self.verification_keys, + listener: BoundaryListener::TlsTcp { + address: self.boundary_listener, + tls: self.sandbox_tls, + }, + resource_claims: resource_claims.clone(), + resource_claim_files: BTreeMap::from([( + "kubernetes.workload_pod_uid".to_string(), + self.workload_pod_uid_path, + )]), + workload_identity: self.workload_identity.clone(), + driver_fence: driver_fence.clone(), + child_env: self.child_env, + }, + runtime_descriptor: SandboxRuntimeDescriptor { + boundary_id: self.boundary_id, + generation: self.generation, + session_id: self.session_id, + workload_identity: self.workload_identity, + transport: SandboxTransport::Tcp { + authority: self.control_authority, + addresses: vec![self.control_address], + }, + tls: self.supervisor_tls, + host_gateway_ip: self.host_gateway_ip, + resource_claims, + driver_fence, + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn spec() -> KubernetesSandboxRuntimeBoundarySpec { + KubernetesSandboxRuntimeBoundarySpec { + boundary_id: "sandbox-1".to_string(), + generation: "generation-1".to_string(), + session_id: openshell_core::SandboxSessionId::new(), + session_rotation: openshell_core::jwt::SessionRotation::new(1).unwrap(), + gateway_id: "gateway-1".to_string(), + verification_keys: vec![GatewayVerificationKey { + key_id: "key-1".to_string(), + public_key_pem: "public-key".to_string(), + }], + namespace_uid: "namespace-uid".to_string(), + sandbox_resource_uid: "sandbox-resource-uid".to_string(), + workload_pod_uid: "pod-uid".to_string(), + workload_pod_uid_path: PathBuf::from("/.openshell/pod-identity/uid"), + supervisor_pod_uid: "supervisor-pod-uid".to_string(), + egress_policy_uid: "network-policy-uid".to_string(), + egress_policy_resource_version: "1945".to_string(), + boundary_listener: "0.0.0.0:5500".parse().expect("valid listener"), + control_authority: "os-boundary-sandbox.default.svc:5500".to_string(), + control_address: "10.42.0.7:5500".parse().expect("valid target"), + sandbox_tls: SandboxTlsServerConfig { + certificate_chain_path: PathBuf::from("/run/boundary/tls.crt"), + private_key_path: PathBuf::from("/run/boundary/tls.key"), + }, + supervisor_tls: SandboxTlsClientConfig { + server_name: "boundary.sandbox.openshell".to_string(), + trust_anchor_pem: "test-ca".to_string(), + }, + host_gateway_ip: Some("10.42.0.1".parse().expect("valid gateway IP")), + workload_identity: ResolvedWorkloadIdentity::new( + 1000, + 1000, + vec![1000], + "kubernetes-config".to_string(), + "sandbox:sandbox-resource-uid".to_string(), + ) + .unwrap(), + child_env: HashMap::new(), + } + } + + #[test] + fn provisioning_binds_identical_kubernetes_resource_claims() { + let provisioned = spec().provision(); + + assert_eq!( + provisioned.boundary_config.resource_claims, + provisioned.runtime_descriptor.resource_claims + ); + assert_eq!( + provisioned.runtime_descriptor.resource_claims["kubernetes.sandbox_resource_uid"], + "sandbox-resource-uid" + ); + assert_eq!( + provisioned.runtime_descriptor.resource_claims["kubernetes.egress_policy_resource_version"], + "1945" + ); + assert_eq!( + provisioned.boundary_config.driver_fence, + provisioned.runtime_descriptor.driver_fence + ); + assert!( + provisioned + .runtime_descriptor + .driver_fence + .validate() + .is_ok() + ); + } + + #[test] + fn provisioning_uses_one_shared_tcp_protocol_across_pods() { + let provisioned = spec().provision(); + + assert_eq!( + provisioned.boundary_config.listener, + BoundaryListener::TlsTcp { + address: "0.0.0.0:5500".parse().expect("valid listener"), + tls: SandboxTlsServerConfig { + certificate_chain_path: PathBuf::from("/run/boundary/tls.crt"), + private_key_path: PathBuf::from("/run/boundary/tls.key"), + }, + } + ); + assert_eq!( + provisioned.runtime_descriptor.transport, + SandboxTransport::Tcp { + authority: "os-boundary-sandbox.default.svc:5500".to_string(), + addresses: vec!["10.42.0.7:5500".parse().expect("valid target")], + } + ); + assert_eq!( + provisioned.runtime_descriptor.tls, + SandboxTlsClientConfig { + server_name: "boundary.sandbox.openshell".to_string(), + trust_anchor_pem: "test-ca".to_string(), + } + ); + } + + #[test] + fn network_fence_denies_all_workload_initiated_egress() { + let fence = KubernetesSandboxRuntimeNetworkFenceSpec { + namespace: "sandbox-ns".to_string(), + policy_name: "openshell-boundary-sandbox-1".to_string(), + supervisor_policy_name: "openshell-sandbox-supervisors".to_string(), + boundary_port: 5500, + } + .provision(); + + let policy_spec = fence.workload_policy.spec.expect("policy has a spec"); + assert_eq!( + policy_spec.policy_types, + Some(vec!["Ingress".to_string(), "Egress".to_string()]) + ); + assert_eq!(policy_spec.egress, Some(Vec::new())); + assert_eq!( + policy_spec.pod_selector.match_labels, + Some(fence.workload_labels) + ); + } + + #[test] + fn network_fence_allows_namespace_supervisors_to_boundary_port() { + let fence = KubernetesSandboxRuntimeNetworkFenceSpec { + namespace: "sandbox-ns".to_string(), + policy_name: "openshell-boundary-sandbox-1".to_string(), + supervisor_policy_name: "openshell-sandbox-supervisors".to_string(), + boundary_port: 5500, + } + .provision(); + + let policy_spec = fence.workload_policy.spec.expect("policy has a spec"); + let ingress = policy_spec + .ingress + .expect("policy has ingress rules") + .pop() + .expect("policy has one ingress rule"); + let peer = ingress + .from + .expect("rule has peers") + .pop() + .expect("rule has one peer"); + assert_eq!( + peer.pod_selector + .expect("peer has a pod selector") + .match_labels, + Some(fence.control_labels) + ); + assert!(peer.namespace_selector.is_none()); + + let port = ingress + .ports + .expect("rule has ports") + .pop() + .expect("rule has one port"); + assert_eq!(port.protocol.as_deref(), Some("TCP")); + assert_eq!(port.port, Some(IntOrString::Int(5500))); + } + + #[test] + fn network_fence_keeps_supervisor_egress_available() { + let fence = KubernetesSandboxRuntimeNetworkFenceSpec { + namespace: "sandbox-ns".to_string(), + policy_name: "openshell-sandbox-workloads".to_string(), + supervisor_policy_name: "openshell-sandbox-supervisors".to_string(), + boundary_port: 5500, + } + .provision(); + + let policy_spec = fence + .supervisor_policy + .spec + .expect("supervisor policy has a spec"); + assert_eq!(policy_spec.policy_types, Some(vec!["Egress".to_string()])); + assert_eq!( + policy_spec.pod_selector.match_labels, + Some(fence.control_labels) + ); + assert_eq!( + policy_spec.egress, + Some(vec![NetworkPolicyEgressRule::default()]) + ); + } +} diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index 28d3c77a7d..bb861308a2 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -4,13 +4,14 @@ pub mod config; pub mod driver; pub mod grpc; +pub mod isolation; pub mod otel_tracing; +mod sandbox_runtime; pub use config::{ - AppArmorProfile, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, - DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesSidecarConfig, - ManagedSshIngressConfig, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, - managed_namespace_prefix, + DEFAULT_GATEWAY_ID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE, + KubernetesComputeConfig, KubernetesImagePullPolicy, KubernetesSandboxRuntimeConfig, + ManagedSshIngressConfig, WorkspaceMode, managed_namespace_prefix, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index d949d6c72c..fbcaf1d5f6 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -8,13 +8,12 @@ use std::net::SocketAddr; use std::path::PathBuf; use tracing::info; +use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; -use openshell_core::{ImagePullPolicy, VERSION}; use openshell_driver_kubernetes::{ - AppArmorProfile, ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, - DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, KubernetesComputeConfig, KubernetesComputeDriver, - KubernetesSidecarConfig, ManagedSshIngressConfig, SupervisorSideloadMethod, SupervisorTopology, - WorkspaceMode, + ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, + KubernetesComputeConfig, KubernetesComputeDriver, KubernetesImagePullPolicy, + KubernetesSandboxRuntimeConfig, ManagedSshIngressConfig, WorkspaceMode, }; #[derive(Parser, Debug)] @@ -72,7 +71,7 @@ struct Args { sandbox_image: Option, #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY")] - sandbox_image_pull_policy: Option, + sandbox_image_pull_policy: Option, #[arg( long, @@ -94,10 +93,8 @@ struct Args { )] managed_ssh_gateway_pod_selector: Vec, - /// Gateway callback endpoint reachable from sandbox pods. Kubernetes - /// service topology cannot be inferred from the sandbox namespace. #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] - grpc_endpoint: String, + grpc_endpoint: Option, #[arg( long, @@ -112,37 +109,31 @@ struct Args { #[arg(long, env = "OPENSHELL_HOST_GATEWAY_IP")] host_gateway_ip: Option, + #[arg(long, env = "OPENSHELL_SANDBOX_RUNTIME_IMAGE")] + sandbox_runtime_image: Option, + + #[arg(long, env = "OPENSHELL_SANDBOX_RUNTIME_IMAGE_PULL_POLICY")] + sandbox_runtime_image_pull_policy: Option, + #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] supervisor_image: Option, #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY")] - supervisor_image_pull_policy: Option, + supervisor_image_pull_policy: Option, #[arg( long, - env = "OPENSHELL_SUPERVISOR_SIDELOAD_METHOD", - default_value = "image-volume" - )] - supervisor_sideload_method: SupervisorSideloadMethod, - - #[arg(long, env = "OPENSHELL_K8S_TOPOLOGY", default_value = "combined")] - topology: SupervisorTopology, - - #[arg( - long = "sidecar-proxy-uid", - alias = "proxy-uid", - env = "OPENSHELL_K8S_SIDECAR_PROXY_UID", - default_value_t = DEFAULT_PROXY_UID + env = "OPENSHELL_K8S_SANDBOX_RUNTIME_NETWORK_POLICY_ENFORCED", + default_value_t = false )] - sidecar_proxy_uid: u32, + sandbox_runtime_network_policy_enforced: bool, #[arg( - long = "sidecar-process-binary-aware-network-policy", - env = "OPENSHELL_K8S_SIDECAR_PROCESS_BINARY_AWARE_NETWORK_POLICY", - default_value_t = true, - action = ArgAction::Set + long, + env = "OPENSHELL_K8S_SANDBOX_RUNTIME_BOUNDARY_PORT", + default_value_t = 5500 )] - sidecar_process_binary_aware_network_policy: bool, + sandbox_runtime_boundary_port: u16, /// Corporate HTTP forward proxy for policy-approved TLS CONNECT egress. #[arg(long, env = "OPENSHELL_UPSTREAM_PROXY")] @@ -171,9 +162,6 @@ struct Args { #[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")] enable_user_namespaces: bool, - #[arg(long, env = "OPENSHELL_K8S_APP_ARMOR_PROFILE")] - app_armor_profile: Option, - /// Lifetime (seconds) of the projected `ServiceAccount` token /// kubelet writes into each sandbox pod for the `IssueSandboxToken` /// bootstrap exchange. Kubelet enforces a minimum of 600s; the @@ -257,16 +245,17 @@ async fn main() -> Result<()> { gateway_namespace: args.managed_ssh_gateway_namespace.unwrap_or_default(), gateway_pod_selector: managed_ssh_gateway_pod_selector, }, + sandbox_runtime_image: args + .sandbox_runtime_image + .unwrap_or_else(openshell_core::config::default_sandbox_runtime_image), + sandbox_runtime_image_pull_policy: args.sandbox_runtime_image_pull_policy, supervisor_image: args .supervisor_image .unwrap_or_else(openshell_core::config::default_supervisor_image), supervisor_image_pull_policy: args.supervisor_image_pull_policy, - supervisor_sideload_method: args.supervisor_sideload_method, - topology: args.topology, - sidecar: KubernetesSidecarConfig { - proxy_uid: args.sidecar_proxy_uid, - process_binary_aware_network_policy: args - .sidecar_process_binary_aware_network_policy, + sandbox_runtime: KubernetesSandboxRuntimeConfig { + network_policy_enforced: args.sandbox_runtime_network_policy_enforced, + boundary_port: args.sandbox_runtime_boundary_port, }, https_proxy: args.https_proxy, no_proxy: args.no_proxy, @@ -274,12 +263,11 @@ async fn main() -> Result<()> { proxy_auth_secret_key: args.proxy_auth_secret_key, proxy_auth_allow_insecure: args.proxy_auth_allow_insecure.then_some(true), proxy_connect_by_hostname: args.proxy_connect_by_hostname.then_some(true), - grpc_endpoint: args.grpc_endpoint, + grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), ssh_socket_path: args.sandbox_ssh_socket_path, client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), enable_user_namespaces: args.enable_user_namespaces, - app_armor_profile: args.app_armor_profile, workspace_default_storage_size: std::env::var( "OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE", ) @@ -337,13 +325,6 @@ async fn main() -> Result<()> { mod tests { use super::*; - #[test] - fn requires_explicit_gateway_callback_endpoint() { - let error = Args::try_parse_from(["openshell-driver-kubernetes"]) - .expect_err("Kubernetes service topology must be explicit"); - assert!(error.to_string().contains("--grpc-endpoint")); - } - #[test] fn accepts_gateway_otlp_configuration() { let args = Args::try_parse_from([ @@ -352,8 +333,6 @@ mod tests { "http://collector.example:4317", "--gateway-name", "kubernetes-dev", - "--grpc-endpoint", - "http://openshell.example:8080", ]) .expect("OTLP endpoint should parse"); diff --git a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs new file mode 100644 index 0000000000..2653597ff3 --- /dev/null +++ b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs @@ -0,0 +1,822 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Kubernetes resources that place and protect the `OpenShell` sandbox runtime. + +use std::collections::BTreeMap; +use std::path::Path; + +use k8s_openapi::ByteString; +use k8s_openapi::api::core::v1::{ + CSIVolumeSource, Capabilities, Container, EmptyDirVolumeSource, EnvVar, ExecAction, KeyToPath, + LocalObjectReference, Pod, PodSchedulingGate, PodSecurityContext, PodSpec, Probe, + ProjectedVolumeSource, Secret, SecretVolumeSource, SecurityContext, Service, + ServiceAccountTokenProjection, ServicePort, ServiceSpec, Volume, VolumeMount, VolumeProjection, +}; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference; +use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; +use kube::core::ObjectMeta; +use rcgen::{CertificateParams, DnType, IsCa, KeyPair, KeyUsagePurpose}; + +use crate::isolation::{ + BOUNDARY_PAIR_LABEL, BOUNDARY_ROLE_LABEL, KubernetesSandboxRuntimeNetworkFence, + KubernetesSandboxRuntimeNetworkFenceSpec, +}; + +pub const SANDBOX_SECRET_COMPONENT: &str = "sandbox-bootstrap"; +pub const SUPERVISOR_SECRET_COMPONENT: &str = "supervisor-bootstrap"; +pub const BOUNDARY_CONFIG_KEY: &str = "boundary.json"; +pub const BACKEND_DESCRIPTOR_KEY: &str = "runtime-descriptor.json"; +pub const BOUNDARY_CERTIFICATE_KEY: &str = "tls.crt"; +pub const BOUNDARY_PRIVATE_KEY: &str = "tls.key"; +pub const SUPERVISOR_AUTH_BUNDLE_KEY: &str = "auth.json"; +pub const PROXY_CA_CERTIFICATE_KEY: &str = "proxy-ca.crt"; +pub const PROXY_CA_PRIVATE_KEY: &str = "proxy-ca.key"; +pub const SANDBOX_BOOTSTRAP_INPUT_PATH: &str = "/.openshell/bootstrap-input"; +pub const BOUNDARY_CONFIG_PATH: &str = "/.openshell/state/bootstrap/boundary.json"; +pub const BOUNDARY_CERTIFICATE_PATH: &str = "/.openshell/state/bootstrap/tls.crt"; +pub const BOUNDARY_PRIVATE_KEY_PATH: &str = "/.openshell/state/bootstrap/tls.key"; +pub const BACKEND_DESCRIPTOR_PATH: &str = "/.openshell/supervisor/runtime-descriptor.json"; +pub const SUPERVISOR_AUTH_BUNDLE_PATH: &str = "/.openshell/supervisor/auth.json"; +pub const PROXY_CA_CERTIFICATE_PATH: &str = "/.openshell/supervisor/proxy-ca.crt"; +pub const PROXY_CA_PRIVATE_KEY_PATH: &str = "/.openshell/supervisor/proxy-ca.key"; +pub const CONTROL_HEALTH_SOCKET_PATH: &str = "/run/openshell/health.sock"; +pub const NAMESPACE_WORKLOAD_POLICY_NAME: &str = "openshell-sandbox-workloads"; +pub const NAMESPACE_SUPERVISOR_EGRESS_POLICY_NAME: &str = "openshell-sandbox-supervisors"; + +pub struct ProxyCaMaterial { + pub certificate_pem: String, + pub private_key_pem: String, +} + +pub fn generate_proxy_ca_material() -> Result { + let key = KeyPair::generate().map_err(|error| format!("generate proxy CA key: {error}"))?; + let mut params = CertificateParams::default(); + params.is_ca = IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + params + .distinguished_name + .push(DnType::CommonName, "OpenShell Sandbox CA"); + params + .distinguished_name + .push(DnType::OrganizationName, "OpenShell"); + params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign]; + let certificate = params + .self_signed(&key) + .map_err(|error| format!("generate proxy CA certificate: {error}"))?; + Ok(ProxyCaMaterial { + certificate_pem: certificate.pem(), + private_key_pem: key.serialize_pem(), + }) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SandboxRuntimeNames { + pub sandbox_secret: String, + pub supervisor_secret: String, + pub boundary_service: String, + pub supervisor_pod: String, + pub workload_policy: String, + pub supervisor_policy: String, +} + +impl SandboxRuntimeNames { + #[must_use] + pub fn new(sandbox_id: &str) -> Self { + let suffix = sandbox_id.to_ascii_lowercase(); + Self { + sandbox_secret: format!("os-sandbox-{suffix}"), + supervisor_secret: format!("os-supervisor-{suffix}"), + boundary_service: format!("os-boundary-{suffix}"), + supervisor_pod: format!("os-supervisor-{suffix}"), + workload_policy: NAMESPACE_WORKLOAD_POLICY_NAME.to_string(), + supervisor_policy: NAMESPACE_SUPERVISOR_EGRESS_POLICY_NAME.to_string(), + } + } + + /// Return stable companion names plus generation-specific immutable + /// bootstrap Secret names. + #[must_use] + pub fn for_generation(sandbox_id: &str, generation: &str) -> Self { + let mut names = Self::new(sandbox_id); + let generation = generation + .chars() + .filter(char::is_ascii_alphanumeric) + .take(12) + .collect::() + .to_ascii_lowercase(); + names.sandbox_secret = format!("{}-{generation}", names.sandbox_secret); + names.supervisor_secret = format!("{}-{generation}", names.supervisor_secret); + names + } +} + +#[must_use] +pub fn pair_label_value(sandbox_id: &str) -> String { + sandbox_id.to_ascii_lowercase() +} + +#[must_use] +pub fn workload_fence( + namespace: &str, + names: &SandboxRuntimeNames, + boundary_port: u16, +) -> KubernetesSandboxRuntimeNetworkFence { + KubernetesSandboxRuntimeNetworkFenceSpec { + namespace: namespace.to_string(), + policy_name: names.workload_policy.clone(), + supervisor_policy_name: names.supervisor_policy.clone(), + boundary_port, + } + .provision() +} + +#[must_use] +pub fn boundary_service( + namespace: &str, + names: &SandboxRuntimeNames, + sandbox_id: &str, + boundary_port: u16, + owner: OwnerReference, +) -> Service { + Service { + metadata: ObjectMeta { + name: Some(names.boundary_service.clone()), + namespace: Some(namespace.to_string()), + owner_references: Some(vec![owner]), + labels: Some(common_labels(sandbox_id, "boundary-service")), + ..Default::default() + }, + spec: Some(ServiceSpec { + selector: Some(pair_labels(sandbox_id, "workload")), + ports: Some(vec![ServicePort { + name: Some("boundary".to_string()), + protocol: Some("TCP".to_string()), + port: i32::from(boundary_port), + target_port: Some(IntOrString::Int(i32::from(boundary_port))), + ..Default::default() + }]), + ..Default::default() + }), + ..Default::default() + } +} + +#[allow(clippy::too_many_arguments, clippy::similar_names)] +pub fn supervisor_pod( + namespace: &str, + names: &SandboxRuntimeNames, + sandbox_id: &str, + sandbox_name: &str, + gateway_id: &str, + supervisor_image: &str, + supervisor_pull_policy: Option, + service_account_name: &str, + control_uid: u32, + control_gid: u32, + image_pull_secrets: &[String], + grpc_endpoint: &str, + client_tls_secret_name: &str, + main_process_spec: &str, + log_level: &str, + sa_token_ttl_secs: i64, + https_proxy: Option<&str>, + no_proxy: Option<&str>, + proxy_auth_secret: Option<(&str, &str)>, + proxy_auth_allow_insecure: bool, + proxy_connect_by_hostname: bool, + provider_spiffe_socket_path: Option<&str>, + owner: OwnerReference, +) -> Result { + let labels = control_labels(sandbox_id, gateway_id); + let mut environment = vec![ + env_var( + "OPENSHELL_ADMITTED_ISOLATION_BACKEND", + crate::isolation::BACKEND_NAME, + ), + env_var("OPENSHELL_ENDPOINT", grpc_endpoint), + env_var("OPENSHELL_SANDBOX_ID", sandbox_id), + env_var("OPENSHELL_SANDBOX", sandbox_name), + env_var("OPENSHELL_MAIN_PROCESS_SPEC", main_process_spec), + env_var( + "OPENSHELL_K8S_SA_TOKEN_FILE", + "/var/run/secrets/openshell/token", + ), + env_var("OPENSHELL_SSH_SOCKET_PATH", "/run/openshell/ssh.sock"), + env_var(openshell_core::sandbox_env::SSH_SOCKET_SHARED, "true"), + env_var("OPENSHELL_PROXY_TLS_DIR", "/run/openshell/proxy-tls"), + env_var( + openshell_core::sandbox_env::PROXY_CA_CERT, + PROXY_CA_CERTIFICATE_PATH, + ), + env_var( + openshell_core::sandbox_env::PROXY_CA_KEY, + PROXY_CA_PRIVATE_KEY_PATH, + ), + env_var("OPENSHELL_LOG_LEVEL", log_level), + env_var( + openshell_core::sandbox_env::TELEMETRY_ENABLED, + openshell_core::telemetry::enabled_env_value(), + ), + env_var( + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + "", + ), + ]; + let mut volume_mounts = vec![ + volume_mount("bootstrap", "/.openshell/supervisor", true), + volume_mount("sa-token", "/var/run/secrets/openshell", true), + volume_mount("run", "/run/openshell", false), + volume_mount("logs", "/var/log", false), + ]; + let mut volumes = vec![ + secret_volume("bootstrap", &names.supervisor_secret, None), + Volume { + name: "sa-token".to_string(), + projected: Some(ProjectedVolumeSource { + default_mode: Some(0o440), + sources: Some(vec![VolumeProjection { + service_account_token: Some(ServiceAccountTokenProjection { + audience: Some("openshell-gateway".to_string()), + expiration_seconds: Some(sa_token_ttl_secs), + path: "token".to_string(), + }), + ..Default::default() + }]), + }), + ..Default::default() + }, + empty_dir_volume("run"), + empty_dir_volume("logs"), + ]; + if !client_tls_secret_name.is_empty() { + environment.extend([ + env_var("OPENSHELL_TLS_CA", "/var/run/secrets/openshell-tls/ca.crt"), + env_var( + "OPENSHELL_TLS_CERT", + "/var/run/secrets/openshell-tls/tls.crt", + ), + env_var( + "OPENSHELL_TLS_KEY", + "/var/run/secrets/openshell-tls/tls.key", + ), + ]); + volume_mounts.push(volume_mount( + "client-tls", + "/var/run/secrets/openshell-tls", + true, + )); + volumes.push(secret_volume("client-tls", client_tls_secret_name, None)); + } + let mut command = vec![ + "/openshell-supervisor".to_string(), + "--backend-descriptor-file".to_string(), + BACKEND_DESCRIPTOR_PATH.to_string(), + "--auth-bundle-file".to_string(), + SUPERVISOR_AUTH_BUNDLE_PATH.to_string(), + "--workdir".to_string(), + "/sandbox".to_string(), + "--health-socket-path".to_string(), + CONTROL_HEALTH_SOCKET_PATH.to_string(), + ]; + if let Some(url) = https_proxy { + command.extend(["--upstream-proxy".to_string(), url.to_string()]); + } + if let Some(hosts) = no_proxy { + command.extend(["--upstream-no-proxy".to_string(), hosts.to_string()]); + } + if proxy_auth_secret.is_some() { + command.extend([ + "--upstream-proxy-auth-file".to_string(), + openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH.to_string(), + ]); + } + if proxy_auth_allow_insecure { + command.push("--upstream-proxy-auth-allow-insecure".to_string()); + } + if proxy_connect_by_hostname { + command.push("--upstream-proxy-connect-by-hostname".to_string()); + } + if let Some((secret_name, secret_key)) = proxy_auth_secret { + let auth_path = Path::new(openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH); + let mount_path = auth_path + .parent() + .and_then(Path::to_str) + .ok_or_else(|| "upstream proxy authentication path has no UTF-8 parent".to_string())?; + let item_path = auth_path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + "upstream proxy authentication path has no UTF-8 file name".to_string() + })?; + volume_mounts.push(volume_mount("upstream-proxy-auth", mount_path, true)); + volumes.push(secret_volume( + "upstream-proxy-auth", + secret_name, + Some(KeyToPath { + key: secret_key.to_string(), + path: item_path.to_string(), + ..Default::default() + }), + )); + } + if let Some(socket_path) = provider_spiffe_socket_path { + environment.push(env_var( + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, + socket_path, + )); + let mount_path = Path::new(socket_path) + .parent() + .and_then(Path::to_str) + .ok_or_else(|| "SPIFFE socket path has no UTF-8 parent".to_string())?; + volume_mounts.push(volume_mount("spiffe-workload-api", mount_path, true)); + volumes.push(Volume { + name: "spiffe-workload-api".to_string(), + csi: Some(CSIVolumeSource { + driver: "csi.spiffe.io".to_string(), + read_only: Some(true), + ..Default::default() + }), + ..Default::default() + }); + } + let mut container = Container { + name: "supervisor".to_string(), + image: Some(supervisor_image.to_string()), + command: Some(command), + termination_message_policy: Some("FallbackToLogsOnError".to_string()), + env: Some(environment), + readiness_probe: Some(Probe { + exec: Some(ExecAction { + command: Some(vec![ + "/openshell-supervisor".to_string(), + "health".to_string(), + "--socket".to_string(), + CONTROL_HEALTH_SOCKET_PATH.to_string(), + ]), + }), + period_seconds: Some(1), + failure_threshold: Some(3), + ..Default::default() + }), + security_context: Some(SecurityContext { + run_as_user: Some(i64::from(control_uid)), + run_as_group: Some(i64::from(control_gid)), + run_as_non_root: Some(true), + read_only_root_filesystem: Some(true), + allow_privilege_escalation: Some(false), + capabilities: Some(Capabilities { + drop: Some(vec!["ALL".to_string()]), + ..Default::default() + }), + ..Default::default() + }), + volume_mounts: Some(volume_mounts), + ..Default::default() + }; + if let Some(policy) = supervisor_pull_policy { + container.image_pull_policy = Some(policy.as_kubernetes_str().to_string()); + } + Ok(Pod { + metadata: ObjectMeta { + name: Some(names.supervisor_pod.clone()), + namespace: Some(namespace.to_string()), + owner_references: Some(vec![owner]), + labels: Some(labels), + annotations: Some(BTreeMap::from([( + "openshell.ai/sandbox-id".to_string(), + sandbox_id.to_string(), + )])), + ..Default::default() + }, + spec: Some(PodSpec { + service_account_name: Some(service_account_name.to_string()), + image_pull_secrets: Some( + image_pull_secrets + .iter() + .map(|name| LocalObjectReference { name: name.clone() }) + .collect(), + ), + automount_service_account_token: Some(false), + scheduling_gates: Some(vec![PodSchedulingGate { + name: "openshell.ai/bootstrap".to_string(), + }]), + security_context: Some(PodSecurityContext { + fs_group: Some(i64::from(control_gid)), + fs_group_change_policy: Some("OnRootMismatch".to_string()), + seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile { + type_: "RuntimeDefault".to_string(), + ..Default::default() + }), + ..Default::default() + }), + restart_policy: Some("Never".to_string()), + containers: vec![container], + volumes: Some(volumes), + ..Default::default() + }), + ..Default::default() + }) +} + +#[allow(clippy::too_many_arguments)] +#[must_use] +pub fn sandbox_bootstrap_secret( + namespace: &str, + names: &SandboxRuntimeNames, + sandbox_id: &str, + boundary_config: Vec, + boundary_certificate: Vec, + boundary_private_key: Vec, + owner: OwnerReference, +) -> Secret { + Secret { + metadata: ObjectMeta { + name: Some(names.sandbox_secret.clone()), + namespace: Some(namespace.to_string()), + owner_references: Some(vec![owner]), + labels: Some(common_labels(sandbox_id, SANDBOX_SECRET_COMPONENT)), + ..Default::default() + }, + data: Some(BTreeMap::from([ + (BOUNDARY_CONFIG_KEY.to_string(), ByteString(boundary_config)), + ( + BOUNDARY_CERTIFICATE_KEY.to_string(), + ByteString(boundary_certificate), + ), + ( + BOUNDARY_PRIVATE_KEY.to_string(), + ByteString(boundary_private_key), + ), + ])), + immutable: Some(true), + type_: Some("Opaque".to_string()), + ..Default::default() + } +} + +#[must_use] +#[allow(clippy::too_many_arguments)] +pub fn supervisor_bootstrap_secret( + namespace: &str, + names: &SandboxRuntimeNames, + sandbox_id: &str, + backend_descriptor: Vec, + supervisor_auth_bundle: Vec, + proxy_ca_certificate: Vec, + proxy_ca_private_key: Vec, + owner: OwnerReference, +) -> Secret { + Secret { + metadata: ObjectMeta { + name: Some(names.supervisor_secret.clone()), + namespace: Some(namespace.to_string()), + owner_references: Some(vec![owner]), + labels: Some(common_labels(sandbox_id, SUPERVISOR_SECRET_COMPONENT)), + ..Default::default() + }, + data: Some(BTreeMap::from([ + ( + BACKEND_DESCRIPTOR_KEY.to_string(), + ByteString(backend_descriptor), + ), + ( + SUPERVISOR_AUTH_BUNDLE_KEY.to_string(), + ByteString(supervisor_auth_bundle), + ), + ( + PROXY_CA_CERTIFICATE_KEY.to_string(), + ByteString(proxy_ca_certificate), + ), + ( + PROXY_CA_PRIVATE_KEY.to_string(), + ByteString(proxy_ca_private_key), + ), + ])), + immutable: Some(true), + type_: Some("Opaque".to_string()), + ..Default::default() + } +} + +#[must_use] +pub fn sandbox_owner_reference( + name: &str, + uid: &str, + api_version: &str, + controller: bool, +) -> OwnerReference { + OwnerReference { + api_version: api_version.to_string(), + kind: "Sandbox".to_string(), + name: name.to_string(), + uid: uid.to_string(), + controller: controller.then_some(true), + // The driver's RBAC intentionally does not permit mutating Sandbox + // finalizers. Kubernetes garbage collection does not require this bit, + // and setting it would make admission fail under + // OwnerReferencesPermissionEnforcement. + block_owner_deletion: Some(false), + } +} + +fn pair_labels(sandbox_id: &str, role: &str) -> BTreeMap { + BTreeMap::from([ + ( + BOUNDARY_PAIR_LABEL.to_string(), + pair_label_value(sandbox_id), + ), + (BOUNDARY_ROLE_LABEL.to_string(), role.to_string()), + ]) +} + +fn common_labels(sandbox_id: &str, component: &str) -> BTreeMap { + BTreeMap::from([ + ( + "openshell.ai/managed-by".to_string(), + "openshell".to_string(), + ), + ( + "openshell.ai/sandbox-id".to_string(), + sandbox_id.to_string(), + ), + ("openshell.ai/component".to_string(), component.to_string()), + ]) +} + +fn control_labels(sandbox_id: &str, gateway_id: &str) -> BTreeMap { + let mut labels = common_labels(sandbox_id, "supervisor"); + labels.extend(pair_labels(sandbox_id, "supervisor")); + labels.insert( + "openshell.ai/gateway-id".to_string(), + gateway_id.to_string(), + ); + labels +} + +fn env_var(name: &str, value: &str) -> EnvVar { + EnvVar { + name: name.to_string(), + value: Some(value.to_string()), + ..Default::default() + } +} + +fn volume_mount(name: &str, mount_path: &str, read_only: bool) -> VolumeMount { + VolumeMount { + name: name.to_string(), + mount_path: mount_path.to_string(), + read_only: read_only.then_some(true), + ..Default::default() + } +} + +fn secret_volume(name: &str, secret_name: &str, item: Option) -> Volume { + Volume { + name: name.to_string(), + secret: Some(SecretVolumeSource { + secret_name: Some(secret_name.to_string()), + default_mode: Some(0o440), + items: item.map(|item| vec![item]), + ..Default::default() + }), + ..Default::default() + } +} + +fn empty_dir_volume(name: &str) -> Volume { + Volume { + name: name.to_string(), + empty_dir: Some(EmptyDirVolumeSource::default()), + ..Default::default() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn owner() -> OwnerReference { + sandbox_owner_reference("demo", "uid-1", "agents.x-k8s.io/v1beta1", false) + } + + #[test] + fn service_selects_only_the_workload_boundary() { + let names = SandboxRuntimeNames::new("4b67c0d0-1111-2222-3333-444444444444"); + let service = boundary_service("sandbox", &names, "pair", 5500, owner()); + assert_eq!( + service.spec.unwrap().selector.unwrap()[BOUNDARY_ROLE_LABEL], + "workload" + ); + } + + #[test] + fn owner_reference_does_not_require_finalizer_mutation_permission() { + assert_eq!(owner().block_owner_deletion, Some(false)); + } + + #[test] + fn supervisor_pod_is_gated_non_restarting_and_unprivileged() { + let names = SandboxRuntimeNames::new("pair"); + let pod = supervisor_pod( + "sandbox", + &names, + "pair", + "demo", + "gateway", + "supervisor:latest", + Some(crate::KubernetesImagePullPolicy::IfNotPresent), + "sandbox-sa", + 1000, + 1000, + &["registry-credentials".to_string()], + "https://gateway:8080", + "client-tls", + "{}", + "info", + 600, + None, + None, + None, + false, + false, + None, + owner(), + ) + .expect("render supervisor Pod"); + let pod_spec = pod.spec.as_ref().expect("Pod spec"); + assert_eq!( + pod.metadata + .owner_references + .as_ref() + .expect("owner references")[0] + .controller, + None + ); + let container = &pod_spec.containers[0]; + assert_eq!(container.image_pull_policy.as_deref(), Some("IfNotPresent")); + assert_eq!(pod_spec.automount_service_account_token, Some(false)); + assert_eq!(pod_spec.restart_policy.as_deref(), Some("Never")); + assert_eq!( + pod_spec + .scheduling_gates + .as_ref() + .and_then(|gates| gates.first()) + .map(|gate| gate.name.as_str()), + Some("openshell.ai/bootstrap") + ); + assert_eq!( + pod_spec + .image_pull_secrets + .as_ref() + .expect("image pull secrets")[0] + .name + .as_str(), + "registry-credentials" + ); + let pod_security = pod_spec + .security_context + .as_ref() + .expect("Pod security context"); + assert_eq!(pod_security.fs_group, Some(1000)); + assert_eq!( + pod_security + .seccomp_profile + .as_ref() + .map(|profile| profile.type_.as_str()), + Some("RuntimeDefault") + ); + let container_security = container + .security_context + .as_ref() + .expect("container security context"); + assert_eq!(container_security.run_as_user, Some(1000)); + assert_eq!(container_security.run_as_non_root, Some(true)); + assert_eq!(container_security.read_only_root_filesystem, Some(true)); + assert_eq!( + container_security + .capabilities + .as_ref() + .and_then(|capabilities| capabilities.drop.as_ref()), + Some(&vec!["ALL".to_string()]) + ); + assert_eq!( + container + .readiness_probe + .as_ref() + .and_then(|probe| probe.exec.as_ref()) + .and_then(|exec| exec.command.as_ref()), + Some(&vec![ + "/openshell-supervisor".to_string(), + "health".to_string(), + "--socket".to_string(), + CONTROL_HEALTH_SOCKET_PATH.to_string(), + ]) + ); + let command = container.command.as_ref().unwrap(); + assert!( + command + .windows(2) + .any(|args| args == ["--health-socket-path", CONTROL_HEALTH_SOCKET_PATH]) + ); + let env = container.env.as_ref().unwrap(); + let env_value = |name: &str| { + env.iter() + .find(|variable| variable.name == name) + .and_then(|variable| variable.value.as_deref()) + }; + assert_eq!( + env_value(openshell_core::sandbox_env::SSH_SOCKET_SHARED), + Some("true") + ); + assert_eq!( + env_value(openshell_core::sandbox_env::PROXY_CA_CERT), + Some(PROXY_CA_CERTIFICATE_PATH) + ); + assert_eq!( + env_value(openshell_core::sandbox_env::PROXY_CA_KEY), + Some(PROXY_CA_PRIVATE_KEY_PATH) + ); + let mount = container + .volume_mounts + .as_ref() + .unwrap() + .iter() + .find(|mount| mount.name == "bootstrap") + .expect("durable supervisor material is mounted into supervisor"); + assert_eq!(mount.mount_path, "/.openshell/supervisor"); + assert_eq!(mount.read_only, Some(true)); + } + + #[test] + fn bootstrap_secrets_are_immutable_and_split_by_trust_domain() { + let names = SandboxRuntimeNames::new("pair"); + let sandbox = sandbox_bootstrap_secret( + "sandbox", + &names, + "pair", + Vec::new(), + Vec::new(), + Vec::new(), + owner(), + ); + assert_eq!( + sandbox.metadata.labels.as_ref().unwrap()["openshell.ai/component"], + SANDBOX_SECRET_COMPONENT + ); + assert_eq!(sandbox.immutable, Some(true)); + let sandbox_keys = sandbox + .data + .unwrap() + .into_keys() + .collect::>(); + assert_eq!( + sandbox_keys, + std::collections::BTreeSet::from([ + BOUNDARY_CERTIFICATE_KEY.to_string(), + BOUNDARY_CONFIG_KEY.to_string(), + BOUNDARY_PRIVATE_KEY.to_string(), + ]) + ); + + let supervisor = supervisor_bootstrap_secret( + "sandbox", + &names, + "pair", + Vec::new(), + Vec::new(), + Vec::new(), + Vec::new(), + owner(), + ); + assert_eq!( + supervisor.metadata.labels.as_ref().unwrap()["openshell.ai/component"], + SUPERVISOR_SECRET_COMPONENT + ); + assert_eq!(supervisor.immutable, Some(true)); + let supervisor_keys = supervisor + .data + .unwrap() + .into_keys() + .collect::>(); + assert_eq!( + supervisor_keys, + std::collections::BTreeSet::from([ + PROXY_CA_CERTIFICATE_KEY.to_string(), + PROXY_CA_PRIVATE_KEY.to_string(), + SUPERVISOR_AUTH_BUNDLE_KEY.to_string(), + BACKEND_DESCRIPTOR_KEY.to_string(), + ]) + ); + } + + #[test] + fn generated_proxy_ca_material_is_pem_encoded() { + let material = generate_proxy_ca_material().unwrap(); + assert!( + material + .certificate_pem + .starts_with("-----BEGIN CERTIFICATE-----") + ); + assert!(material.private_key_pem.contains("PRIVATE KEY")); + } +} diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 92bd2a9db8..6979507eac 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -35,7 +35,8 @@ where Helm cannot discover cluster APIs. ## Install on Kubernetes ```shell -helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true ``` ## Install on OpenShift @@ -43,14 +44,12 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -n openshell \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set server.disableTls=true \ --set podSecurityContext.fsGroup=null \ --set securityContext.runAsUser=null @@ -107,6 +106,7 @@ Then install the chart pointing at that Secret: ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ -n openshell \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set workload.kind=deployment \ --set server.externalDbSecret=my-pg-credentials ``` @@ -223,6 +223,9 @@ discovery endpoint or its TLS CA. | probes.startup.timeoutSeconds | int | `1` | Startup probe timeout, in seconds. | | replicaCount | int | `1` | Number of OpenShell gateway replicas. Values greater than 1 require server.externalDbSecret because the default SQLite backend is per pod. | | resources | object | `{}` | Gateway pod resource requests and limits. | +| sandboxRuntime.image.pullPolicy | string | `""` | Sandbox runtime image pull policy. Defaults to the gateway image pull policy when empty. | +| sandboxRuntime.image.repository | string | `"ghcr.io/nvidia/openshell/sandbox"` | Sandbox runtime image repository. Changing it uses the effective gateway image tag unless tag is also set. | +| sandboxRuntime.image.tag | string | `""` | Sandbox runtime image tag override. Empty uses the version pinned into the gateway unless repository is changed. | | sandboxServiceAccount.annotations | object | `{}` | Annotations to add to the generated sandbox service account. | | sandboxServiceAccount.create | bool | `true` | Create a service account for sandbox pods. | | sandboxServiceAccount.name | string | `""` | Existing service account name for sandbox pods when sandboxServiceAccount.create is false. | @@ -230,7 +233,6 @@ discovery endpoint or its TLS CA. | securityContext.capabilities.drop | list | `["ALL"]` | Linux capabilities dropped from the gateway container. | | securityContext.runAsNonRoot | bool | `true` | Require the gateway container to run as a non-root user. | | securityContext.runAsUser | int | `1000` | UID assigned to the gateway container. | -| server.appArmorProfile | string | `"Unconfined"` | Kubernetes AppArmor profile requested for sandbox agent containers. Default Unconfined avoids runtime/default AppArmor blocking the supervisor's network namespace mount setup on AppArmor-enabled nodes. Set to "" to omit the field, "RuntimeDefault" to force the runtime default profile, or "Localhost/profile-name" for an operator-managed localhost profile. | | server.auth.allowUnauthenticatedUsers | bool | `false` | UNSAFE: accept unauthenticated CLI/user requests as a local developer principal. Intended only for trusted local Skaffold/k3d development or a fully trusted fronting proxy. Leave false for shared or production clusters. | | server.credentialDrivers.kubernetesSecrets.allowReferenceNamespace | bool | `false` | Deprecated compatibility field. Credential storage no longer supports user-authored namespace references. | | server.credentialDrivers.kubernetesSecrets.enabled | bool | `false` | Enable the in-tree Kubernetes Secret credential driver. WARNING: The RBAC Role grants read/write access to ALL Secrets in the configured namespace. Use a dedicated namespace to limit blast radius. | @@ -300,10 +302,8 @@ discovery endpoint or its TLS CA. | supervisor.image.pullPolicy | string | `nil` | Sandbox supervisor pull policy. Leave unset to use the Kubernetes image default. Prefer always, if_not_present, or never; the chart also accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. | | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. | | supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. | -| supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar runs as proxyUid, drops the extra /proc inspection capabilities, and enforces endpoint/L7 policy without matching policy.binaries. | -| supervisor.sidecar.proxyUid | int | `1337` | UID for relaxed long-running network sidecars in sidecar topology. Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants the required /proc inspection capabilities into the effective set. The network init container installs nftables rules that exempt the effective sidecar UID. | -| supervisor.sideloadMethod | string | `""` | How the supervisor binary is delivered into sandbox pods. Empty (default) = auto-detect from cluster version: K8s >= v1.35 -> "image-volume" (ImageVolume enabled by default; GA in v1.36) K8s < v1.35 -> "init-container" (copies via init container + emptyDir) On K8s v1.33-v1.34 with the ImageVolume feature gate manually enabled, set this to "image-volume" explicitly. | -| supervisor.topology | string | `"combined"` | Supervisor pod topology for Kubernetes sandboxes. "combined" runs the current single supervisor container in the agent pod. "sidecar" runs network enforcement in a dedicated sidecar and the process supervisor as a low-capability wrapper in the agent container. | +| supervisor.sandboxRuntime.boundaryPort | int | `5500` | Workload boundary TLS listener port. | +| supervisor.sandboxRuntime.networkPolicyEnforced | bool | `false` | Required operator acknowledgement that the cluster CNI enforces NetworkPolicy. | | tolerations | list | `[]` | Tolerations for the gateway pod. | | upstreamProxy | object | `{"authAllowInsecure":false,"authSecret":{"key":"","name":""},"connectByHostname":false,"noProxy":"","url":""}` | Operator-owned corporate forward proxy for policy-approved TLS egress from Kubernetes sandboxes. The workload cannot select or override it. | | upstreamProxy.authAllowInsecure | bool | `false` | Required when authSecret is configured because Basic auth to an HTTP proxy is cleartext. | diff --git a/deploy/helm/openshell/README.md.gotmpl b/deploy/helm/openshell/README.md.gotmpl index cf8677741e..5cbded1df7 100644 --- a/deploy/helm/openshell/README.md.gotmpl +++ b/deploy/helm/openshell/README.md.gotmpl @@ -35,7 +35,8 @@ where Helm cannot discover cluster APIs. ## Install on Kubernetes ```shell -helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true ``` ## Install on OpenShift @@ -43,14 +44,12 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -n openshell \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set server.disableTls=true \ --set podSecurityContext.fsGroup=null \ --set securityContext.runAsUser=null @@ -107,6 +106,7 @@ Then install the chart pointing at that Secret: ```bash helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version \ -n openshell \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set workload.kind=deployment \ --set server.externalDbSecret=my-pg-credentials ``` diff --git a/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml b/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml index 35532440b2..70fdec8d7b 100644 --- a/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml +++ b/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml @@ -3,9 +3,6 @@ # The Kubernetes corporate-proxy e2e wrapper supplies the generated proxy URL # and creates `openshell-e2e-proxy-auth` before Helm installs the gateway. -supervisor: - topology: sidecar - upstreamProxy: authSecret: name: openshell-e2e-proxy-auth diff --git a/deploy/helm/openshell/ci/values-keycloak.yaml b/deploy/helm/openshell/ci/values-keycloak.yaml index cc6ca658bf..74e38b6043 100644 --- a/deploy/helm/openshell/ci/values-keycloak.yaml +++ b/deploy/helm/openshell/ci/values-keycloak.yaml @@ -8,7 +8,8 @@ # # Then layer this file on top of values.yaml when deploying: # helm upgrade --install openshell . \ -# -f values.yaml -f ci/values-skaffold.yaml -f ci/values-keycloak.yaml +# -f values.yaml -f ci/values-skaffold.yaml -f ci/values-keycloak.yaml \ +# --set supervisor.sandboxRuntime.networkPolicyEnforced=true # # Or add this file to skaffold.yaml valuesFiles for iterative dev. # diff --git a/deploy/helm/openshell/ci/values-openshift-scc.yaml b/deploy/helm/openshell/ci/values-openshift-scc.yaml index b7f37be6e0..8f1a8d07d6 100644 --- a/deploy/helm/openshell/ci/values-openshift-scc.yaml +++ b/deploy/helm/openshell/ci/values-openshift-scc.yaml @@ -4,7 +4,8 @@ # OpenShift SCC compatibility overlay. Removes the hardcoded runAsUser and # fsGroup so that OpenShift's restricted-v2 SCC can inject the namespace- # assigned UID/GID range. Layer after values.yaml: -# helm install openshell deploy/helm/openshell -f ci/values-openshift-scc.yaml +# helm install openshell deploy/helm/openshell -f ci/values-openshift-scc.yaml \ +# --set supervisor.sandboxRuntime.networkPolicyEnforced=true # # The e2e Kubernetes harness applies this automatically when it detects an # OpenShift cluster (route.openshift.io API present). diff --git a/deploy/helm/openshell/ci/values-sandbox-runtime.yaml b/deploy/helm/openshell/ci/values-sandbox-runtime.yaml new file mode 100644 index 0000000000..471ecc3098 --- /dev/null +++ b/deploy/helm/openshell/ci/values-sandbox-runtime.yaml @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +supervisor: + sandboxRuntime: + networkPolicyEnforced: true diff --git a/deploy/helm/openshell/ci/values-sidecar-kata.yaml b/deploy/helm/openshell/ci/values-sidecar-kata.yaml deleted file mode 100644 index 2e23a1009e..0000000000 --- a/deploy/helm/openshell/ci/values-sidecar-kata.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# CI/dev overlay for exercising the Kubernetes supervisor sidecar topology under -# a Kata RuntimeClass. -# -# Use with e2e/with-kube-gateway.sh by setting: -# OPENSHELL_E2E_KUBE_EXTRA_VALUES=deploy/helm/openshell/ci/values-sidecar-kata.yaml -# The e2e wrapper supplies the image repository and tag through OPENSHELL_REGISTRY -# and IMAGE_TAG for existing-cluster runs. - -supervisor: - # Use the sidecar topology under Kata so network enforcement runs in the - # sidecar and the sandbox agent container stays low-privilege. - topology: sidecar - sidecar: - # Keep strict process/binary-aware network policy enabled for the Kata - # validation path. Set this false only when intentionally validating the - # documented endpoint/L7-only downgrade mode. - processBinaryAwareNetworkPolicy: true - -# Kata validation clusters normally install this RuntimeClass. -server: - defaultRuntimeClassName: kata-qemu diff --git a/deploy/helm/openshell/ci/values-sidecar.yaml b/deploy/helm/openshell/ci/values-sidecar.yaml deleted file mode 100644 index ba8dc50ef1..0000000000 --- a/deploy/helm/openshell/ci/values-sidecar.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# CI/dev overlay for exercising the Kubernetes supervisor sidecar topology. -# -# Merge after values.yaml and ci/values-skaffold.yaml: -# helm install ... -f values.yaml -f ci/values-skaffold.yaml -f ci/values-sidecar.yaml -# -# Or set: -# OPENSHELL_E2E_KUBE_EXTRA_VALUES=deploy/helm/openshell/ci/values-sidecar.yaml -# before running `mise run e2e:kubernetes`. -supervisor: - topology: sidecar - sidecar: - # The strict sidecar default requires cross-container /proc identity access. - # CI/dev e2e uses the explicit downgraded mode so endpoint and L7 policy - # coverage remains runnable on local k3d while that path is hardened. - processBinaryAwareNetworkPolicy: false diff --git a/deploy/helm/openshell/skaffold.yaml b/deploy/helm/openshell/skaffold.yaml index f10da7c7ff..35cd1fc9c4 100644 --- a/deploy/helm/openshell/skaffold.yaml +++ b/deploy/helm/openshell/skaffold.yaml @@ -136,8 +136,6 @@ deploy: # To enable SPIFFE/SPIRE provider token grants (requires the # spire-crds and spire releases above): #- ci/values-spire.yaml - # To exercise the Kubernetes supervisor sidecar topology: - #- ci/values-sidecar.yaml # To test multi-replica external PostgreSQL behavior: #- ci/values-high-availability.yaml setValueTemplates: @@ -146,20 +144,6 @@ deploy: supervisor.image.repository: '{{.IMAGE_REPO_openshell_supervisor}}' supervisor.image.tag: '{{.IMAGE_TAG_openshell_supervisor}}' profiles: - - name: sidecar - patches: - - op: add - path: /deploy/helm/releases/0/valuesFiles/- - value: ci/values-sidecar.yaml - - name: sidecar-mtls - patches: - - op: add - path: /deploy/helm/releases/0/valuesFiles/- - value: ci/values-sidecar.yaml - - op: add - path: /deploy/helm/releases/0/setValues - value: - server.disableTls: "false" - name: credential-driver-kubernetes-secrets patches: - op: add diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 98243627dd..98e9c67ed9 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -91,6 +91,25 @@ so a released chart automatically pulls the matching image without extra overrid {{- printf "%s:%s" .Values.image.repository (.Values.image.tag | default .Chart.AppVersion) }} {{- end }} +{{/* Official sandbox runtime repository used by the gateway's built-in default. */}} +{{- define "openshell.defaultSandboxRuntimeRepository" -}} +ghcr.io/nvidia/openshell/sandbox +{{- end }} + +{{/* Whether Helm must propagate a sandbox runtime image override. */}} +{{- define "openshell.sandboxRuntimeImageOverrideEnabled" -}} +{{- $defaultRepository := include "openshell.defaultSandboxRuntimeRepository" . -}} +{{- $repository := .Values.sandboxRuntime.image.repository | default $defaultRepository -}} +{{- if or (ne $repository $defaultRepository) .Values.sandboxRuntime.image.tag -}}true{{- end -}} +{{- end }} + +{{/* Sandbox runtime image override. */}} +{{- define "openshell.sandboxRuntimeImage" -}} +{{- $repository := .Values.sandboxRuntime.image.repository | default (include "openshell.defaultSandboxRuntimeRepository" .) -}} +{{- $tag := .Values.sandboxRuntime.image.tag | default .Values.image.tag | default .Chart.AppVersion -}} +{{- printf "%s:%s" $repository $tag }} +{{- end }} + {{/* Official supervisor repository used by the gateway's built-in default. */}} {{- define "openshell.defaultSupervisorRepository" -}} ghcr.io/nvidia/openshell/supervisor @@ -194,25 +213,6 @@ the in-cluster Service DNS, release namespace, service port, and disableTls flag — so the default value works for any release name or namespace without override. */}} -{{/* -Supervisor sideload method. When supervisor.sideloadMethod is set, use it -verbatim. Otherwise auto-detect from the cluster version: the ImageVolume -feature gate is enabled by default starting in K8s v1.35 (GA in v1.36). -Clusters on v1.33-v1.34 can opt in by setting sideloadMethod explicitly -after enabling the feature gate. -*/}} -{{- define "openshell.supervisorSideloadMethod" -}} -{{- if .Values.supervisor.sideloadMethod -}} -{{- .Values.supervisor.sideloadMethod -}} -{{- else -}} -{{- if semverCompare ">=1.35-0" .Capabilities.KubeVersion.Version -}} -image-volume -{{- else -}} -init-container -{{- end -}} -{{- end -}} -{{- end }} - {{- define "openshell.grpcEndpoint" -}} {{- if .Values.server.grpcEndpoint -}} {{- .Values.server.grpcEndpoint -}} diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 5d328a1261..1f8be73ff5 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -71,7 +71,12 @@ rules: resources: - pods verbs: + - create + - delete - get + - list + - patch + - watch {{- end }} {{- $copiedSecretNames := list }} {{- if and (ne $workspaceMode "shared") (not .Values.server.disableTls) }} @@ -84,6 +89,17 @@ rules: {{- end }} {{- end }} {{- end }} + {{- if ne $workspaceMode "shared" }} + - apiGroups: [""] + resources: ["services"] + verbs: ["create", "delete", "get"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] + - apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["create", "get"] + {{- end }} {{- $copiedSecretNames = uniq $copiedSecretNames }} {{- if $copiedSecretNames }} # Copy only explicitly configured TLS and image-pull Secrets into workspace diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 2c2b8216ee..5265f9b46b 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -138,6 +138,9 @@ data: [openshell.drivers.kubernetes] namespace = {{ include "openshell.sandboxNamespace" . | quote }} default_image = {{ .Values.server.sandboxImage | quote }} + {{- if include "openshell.sandboxRuntimeImageOverrideEnabled" . }} + sandbox_runtime_image = {{ include "openshell.sandboxRuntimeImage" . | quote }} + {{- end }} {{- if include "openshell.supervisorImageOverrideEnabled" . }} supervisor_image = {{ include "openshell.supervisorImage" . | quote }} {{- end }} @@ -160,8 +163,6 @@ data: {{- if .Values.server.drivers.kubernetes.operatorNamespaceFile }} operator_namespace_file = {{ .Values.server.drivers.kubernetes.operatorNamespaceFile | quote }} {{- end }} - supervisor_sideload_method = {{ include "openshell.supervisorSideloadMethod" . | quote }} - topology = {{ .Values.supervisor.topology | default "combined" | quote }} sa_token_ttl_secs = {{ .Values.server.sandboxJwt.k8sSaTokenTtlSecs | default 3600 }} {{- if .Values.upstreamProxy.url }} https_proxy = {{ .Values.upstreamProxy.url | quote }} @@ -205,21 +206,21 @@ data: {{- if .Values.server.defaultRuntimeClassName }} default_runtime_class_name = {{ .Values.server.defaultRuntimeClassName | quote }} {{- end }} - {{- if .Values.server.appArmorProfile }} - app_armor_profile = {{ .Values.server.appArmorProfile | quote }} - {{- end }} {{- if .Values.supervisor.image.pullPolicy }} supervisor_image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.supervisor.image.pullPolicy | quote }} {{- end }} + {{- if .Values.sandboxRuntime.image.pullPolicy }} + sandbox_runtime_image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.sandboxRuntime.image.pullPolicy | quote }} + {{- end }} [openshell.drivers.kubernetes.managed_ssh_ingress] enabled = {{ .Values.networkPolicy.enabled }} gateway_namespace = {{ .Release.Namespace | quote }} gateway_pod_selector = { "app.kubernetes.io/name" = {{ include "openshell.name" . | quote }}, "app.kubernetes.io/instance" = {{ .Release.Name | quote }} } - [openshell.drivers.kubernetes.sidecar] - proxy_uid = {{ .Values.supervisor.sidecar.proxyUid | default 1337 }} - process_binary_aware_network_policy = {{ .Values.supervisor.sidecar.processBinaryAwareNetworkPolicy }} + [openshell.drivers.kubernetes.sandbox_runtime] + network_policy_enforced = {{ .Values.supervisor.sandboxRuntime.networkPolicyEnforced }} + boundary_port = {{ .Values.supervisor.sandboxRuntime.boundaryPort | default 5500 }} {{- if not $credentialDrivers }} diff --git a/deploy/helm/openshell/templates/network-policy-ack.yaml b/deploy/helm/openshell/templates/network-policy-ack.yaml new file mode 100644 index 0000000000..780d4314c9 --- /dev/null +++ b/deploy/helm/openshell/templates/network-policy-ack.yaml @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +{{- if not .Values.supervisor.sandboxRuntime.networkPolicyEnforced }} +{{- fail "supervisor.sandboxRuntime.networkPolicyEnforced must be true after you verify that the cluster CNI enforces ingress and egress NetworkPolicy in every sandbox namespace" }} +{{- end }} diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index dfd6423615..8e556d064d 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -33,14 +33,28 @@ rules: - watch # Per-sandbox identity: TokenReview authenticates the projected token from # the configured sandbox service account, then the gateway resolves the - # returned pod name and UID to the pod's `openshell.ai/sandbox-id` - # annotation. patch is intentionally NOT granted — the annotation is set - # once at pod create and must remain immutable for the lifetime of the - # sandbox. + # returned pod name and UID to its immutable sandbox labels. The driver also + # creates and deletes the directly managed supervisor Pod. PATCH removes the + # driver-owned bootstrap scheduling gate after immutable bootstrap material + # has been created. - apiGroups: - "" resources: - pods verbs: + - create + - delete - get + - list + - patch + - watch + - apiGroups: [""] + resources: ["services"] + verbs: ["create", "delete", "get"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] + - apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["create", "get"] {{- end }} diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml index 8f16aef584..9eb6071863 100644 --- a/deploy/helm/openshell/tests/clusterrole_test.yaml +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -26,17 +26,23 @@ tests: resources: ["networkpolicies"] verbs: ["get", "create", "patch", "update"] - - it: omits managed NetworkPolicy permissions when isolation is disabled + - it: preserves sandbox-runtime fence permissions when gateway isolation is disabled set: server.drivers.kubernetes.workspaceMode: managed networkPolicy.enabled: false asserts: + - contains: + path: rules + content: + apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["create", "get"] - notContains: path: rules content: apiGroups: ["networking.k8s.io"] resources: ["networkpolicies"] - any: true + verbs: ["get", "create", "patch", "update"] - it: grants broad secret access when credential driver is enabled (operator) set: @@ -63,17 +69,23 @@ tests: resourceNames: ["custom-client-tls"] verbs: ["get", "patch"] - - it: omits operator secret access when TLS and credential storage are disabled + - it: limits operator bootstrap secret access to create when TLS and credential storage are disabled set: server.drivers.kubernetes.workspaceMode: operator server.disableTls: true asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] - notContains: path: rules content: apiGroups: [""] resources: ["secrets"] - any: true + verbs: ["get", "create", "patch", "delete"] - it: restricts managed copies to TLS and configured image-pull secrets set: @@ -128,3 +140,37 @@ tests: apiGroups: [""] resources: ["secrets"] any: true + + - it: grants managed sandbox-runtime companion permissions + set: + server.drivers.kubernetes.workspaceMode: managed + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["pods"] + verbs: ["create", "delete", "get", "list", "patch", "watch"] + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] + + - it: grants operator sandbox-runtime companion permissions + set: + server.drivers.kubernetes.workspaceMode: operator + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["pods"] + verbs: ["create", "delete", "get", "list", "patch", "watch"] + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 9805af8254..03fae997dc 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -237,16 +237,6 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.gateway\][^\[]*?enable_user_namespaces' - - it: renders combined supervisor topology by default under [openshell.drivers.kubernetes] - template: templates/gateway-config.yaml - asserts: - - matchRegex: - path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?topology\s*=\s*"combined"' - - notMatchRegex: - path: data["gateway.toml"] - pattern: 'supervisor[_]topology\s*=' - - it: renders operator-owned upstream proxy settings under the Kubernetes driver template: templates/gateway-config.yaml set: @@ -283,64 +273,60 @@ tests: path: data["gateway.toml"] pattern: 'supervisor_image\s*=' - - it: renders a supervisor tag override with the official repository + - it: uses the gateway built-in sandbox runtime image by default template: templates/gateway-config.yaml - set: - supervisor.image.tag: 1.2.3 asserts: - - matchRegex: + - notMatchRegex: path: data["gateway.toml"] - pattern: 'supervisor_image\s*=\s*"ghcr\.io/nvidia/openshell/supervisor:1\.2\.3"' + pattern: 'sandbox_runtime_image\s*=' - - it: renders a supervisor repository override with the effective gateway tag + - it: renders independent sandbox runtime and supervisor image overrides template: templates/gateway-config.yaml set: - image.tag: gateway-build + sandboxRuntime.image.repository: registry.example.com/openshell/sandbox + sandboxRuntime.image.tag: sandbox-build + sandboxRuntime.image.pullPolicy: Always supervisor.image.repository: registry.example.com/openshell/supervisor + supervisor.image.tag: supervisor-build asserts: - matchRegex: path: data["gateway.toml"] - pattern: 'supervisor_image\s*=\s*"registry\.example\.com/openshell/supervisor:gateway-build"' - - - it: renders complete supervisor repository and tag overrides - template: templates/gateway-config.yaml - set: - supervisor.image.repository: registry.example.com/openshell/supervisor - supervisor.image.tag: supervisor-build - asserts: + pattern: 'sandbox_runtime_image\s*=\s*"registry\.example\.com/openshell/sandbox:sandbox-build"' + - matchRegex: + path: data["gateway.toml"] + pattern: 'sandbox_runtime_image_pull_policy\s*=\s*"always"' - matchRegex: path: data["gateway.toml"] pattern: 'supervisor_image\s*=\s*"registry\.example\.com/openshell/supervisor:supervisor-build"' - - it: renders sidecar supervisor topology under [openshell.drivers.kubernetes] + - it: renders a supervisor tag override with the official repository template: templates/gateway-config.yaml set: - supervisor.topology: sidecar + supervisor.image.tag: 1.2.3 asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?topology\s*=\s*"sidecar"' - - notMatchRegex: - path: data["gateway.toml"] - pattern: 'supervisor[_]topology\s*=' + pattern: 'supervisor_image\s*=\s*"ghcr\.io/nvidia/openshell/supervisor:1\.2\.3"' - - it: renders proxy uid under [openshell.drivers.kubernetes.sidecar] + - it: renders a supervisor repository override with the effective gateway tag template: templates/gateway-config.yaml set: - supervisor.sidecar.proxyUid: 2200 + image.tag: gateway-build + supervisor.image.repository: registry.example.com/openshell/supervisor asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\.sidecar\].*?proxy_uid\s*=\s*2200' + pattern: 'supervisor_image\s*=\s*"registry\.example\.com/openshell/supervisor:gateway-build"' - - it: renders process binary aware network policy under [openshell.drivers.kubernetes.sidecar] + - it: renders complete supervisor repository and tag overrides template: templates/gateway-config.yaml set: - supervisor.sidecar.processBinaryAwareNetworkPolicy: false + supervisor.image.repository: registry.example.com/openshell/supervisor + supervisor.image.tag: supervisor-build asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\.sidecar\].*?process_binary_aware_network_policy\s*=\s*false' + pattern: 'supervisor_image\s*=\s*"registry\.example\.com/openshell/supervisor:supervisor-build"' - it: configures managed SSH isolation with the gateway peer template: templates/gateway-config.yaml @@ -360,22 +346,6 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?image_pull_secrets\s*=\s*\["regcred", "backup-regcred"\]' - - it: renders the default sandbox AppArmor profile under [openshell.drivers.kubernetes] - template: templates/gateway-config.yaml - asserts: - - matchRegex: - path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?app_armor_profile\s*=\s*"Unconfined"' - - - it: omits sandbox AppArmor profile when disabled - template: templates/gateway-config.yaml - set: - server.appArmorProfile: "" - asserts: - - notMatchRegex: - path: data["gateway.toml"] - pattern: 'app_armor_profile\s*=' - - it: does not reuse gateway image pull secrets for sandbox pods template: templates/gateway-config.yaml set: diff --git a/deploy/helm/openshell/tests/network_policy_ack_test.yaml b/deploy/helm/openshell/tests/network_policy_ack_test.yaml new file mode 100644 index 0000000000..12ca399e73 --- /dev/null +++ b/deploy/helm/openshell/tests/network_policy_ack_test.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: NetworkPolicy enforcement acknowledgement +templates: + - templates/network-policy-ack.yaml +tests: + - it: rejects an install without operator acknowledgement + set: + supervisor.sandboxRuntime.networkPolicyEnforced: false + asserts: + - failedTemplate: + errorMessage: supervisor.sandboxRuntime.networkPolicyEnforced must be true after you verify that the cluster CNI enforces ingress and egress NetworkPolicy in every sandbox namespace + + - it: accepts an acknowledged CNI + set: + supervisor.sandboxRuntime.networkPolicyEnforced: true + asserts: + - hasDocuments: + count: 0 diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 8f5bfa9959..35129b47df 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -26,9 +26,17 @@ image: # -- Gateway image tag. Defaults to the chart appVersion when empty. tag: "" -# Supervisor image for the openshell-sandbox binary injected into sandbox pods. -# The default repository and empty tag use the version-pinned image built into -# the gateway. Changing the repository or setting a tag enables a Helm override. +# Trusted workload-side runtime image. +sandboxRuntime: + image: + # -- Sandbox runtime image repository. Changing it uses the effective gateway image tag unless tag is also set. + repository: ghcr.io/nvidia/openshell/sandbox + # -- Sandbox runtime image pull policy. Defaults to the gateway image pull policy when empty. + pullPolicy: "" + # -- Sandbox runtime image tag override. Empty uses the version pinned into the gateway unless repository is changed. + tag: "" + +# Trusted control-side runtime image. supervisor: image: # -- Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. @@ -39,30 +47,11 @@ supervisor: pullPolicy: null # -- Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. tag: "" - # -- How the supervisor binary is delivered into sandbox pods. - # Empty (default) = auto-detect from cluster version: - # K8s >= v1.35 -> "image-volume" (ImageVolume enabled by default; GA in v1.36) - # K8s < v1.35 -> "init-container" (copies via init container + emptyDir) - # On K8s v1.33-v1.34 with the ImageVolume feature gate manually enabled, - # set this to "image-volume" explicitly. - sideloadMethod: "" - # -- Supervisor pod topology for Kubernetes sandboxes. - # "combined" runs the current single supervisor container in the agent pod. - # "sidecar" runs network enforcement in a dedicated sidecar and the process - # supervisor as a low-capability wrapper in the agent container. - topology: "combined" - sidecar: - # -- UID for relaxed long-running network sidecars in sidecar topology. - # Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants - # the required /proc inspection capabilities into the effective set. The - # network init container installs nftables rules that exempt the effective - # sidecar UID. - proxyUid: 1337 - # -- Keep process/binary-aware network policy enabled in sidecar topology. - # When false, the network sidecar runs as proxyUid, drops the extra /proc - # inspection capabilities, and enforces endpoint/L7 policy without matching - # policy.binaries. - processBinaryAwareNetworkPolicy: true + sandboxRuntime: + # -- Required operator acknowledgement that the cluster CNI enforces NetworkPolicy. + networkPolicyEnforced: false + # -- Workload boundary TLS listener port. + boundaryPort: 5500 # -- Operator-owned corporate forward proxy for policy-approved TLS egress # from Kubernetes sandboxes. The workload cannot select or override it. @@ -269,12 +258,6 @@ server: # Linux 5.12+. When enabled, container UID 0 maps to an unprivileged host # UID and capabilities become namespaced. enableUserNamespaces: false - # -- Kubernetes AppArmor profile requested for sandbox agent containers. - # Default Unconfined avoids runtime/default AppArmor blocking the supervisor's - # network namespace mount setup on AppArmor-enabled nodes. Set to "" to omit - # the field, "RuntimeDefault" to force the runtime default profile, or - # "Localhost/profile-name" for an operator-managed localhost profile. - appArmorProfile: "Unconfined" # Kubernetes compute driver settings. drivers: kubernetes: diff --git a/deploy/helm/test-split-ownership.sh b/deploy/helm/test-split-ownership.sh index 30fd7360d4..a7a865363b 100755 --- a/deploy/helm/test-split-ownership.sh +++ b/deploy/helm/test-split-ownership.sh @@ -11,6 +11,7 @@ trap 'rm -rf "${work_dir}"' EXIT helm template openshell "${repo_root}/deploy/helm/openshell" \ --namespace openshell \ --set agentSandbox.preflight.enabled=false \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set workspaceResources.enabled=false \ >"${work_dir}/gateway.yaml" @@ -40,6 +41,7 @@ fi helm template openshell "${repo_root}/deploy/helm/openshell" \ --namespace openshell \ --set agentSandbox.preflight.enabled=false \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set-json workspaceResources=null \ >"${work_dir}/legacy-reuse-values.yaml" diff --git a/docs/kubernetes/ingress.mdx b/docs/kubernetes/ingress.mdx index 51284465d9..deb0da0b6f 100644 --- a/docs/kubernetes/ingress.mdx +++ b/docs/kubernetes/ingress.mdx @@ -57,6 +57,7 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set grpcRoute.enabled=true \ --set grpcRoute.gateway.create=true \ --set grpcRoute.gateway.className=eg @@ -113,6 +114,7 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set grpcRoute.enabled=true \ --set grpcRoute.gateway.create=true \ --set grpcRoute.gateway.className=eg \ diff --git a/docs/kubernetes/managing-certificates.mdx b/docs/kubernetes/managing-certificates.mdx index c4cb07f57e..4ddaf2c1e9 100644 --- a/docs/kubernetes/managing-certificates.mdx +++ b/docs/kubernetes/managing-certificates.mdx @@ -52,6 +52,7 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set certManager.enabled=true ``` @@ -73,6 +74,7 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set certManager.enabled=true \ --set certManager.serverIssuerRef.name=letsencrypt-prod \ --set certManager.serverIssuerRef.kind=ClusterIssuer \ diff --git a/docs/kubernetes/openshift.mdx b/docs/kubernetes/openshift.mdx index 43e7d0338b..ba3199ca1f 100644 --- a/docs/kubernetes/openshift.mdx +++ b/docs/kubernetes/openshift.mdx @@ -3,140 +3,66 @@ # SPDX-License-Identifier: Apache-2.0 title: "OpenShift" sidebar-title: "OpenShift" -description: "Install the OpenShell Helm chart on OpenShift, including the SCC binding and chart overrides required by OpenShift's Security Context Constraints." +description: "Install the OpenShell Helm chart on OpenShift with capability-free sandbox workloads." keywords: "Generative AI, Cybersecurity, Kubernetes, OpenShift, SCC, Security Context Constraints, Helm, Gateway, Installation" position: 6 --- +The Kubernetes driver resolves the UID range assigned to each OpenShift +namespace and renders the sandbox and supervisor with a numeric non-root +identity from that range. OpenShell does not require the `privileged` SCC or any +added Linux capability. + -The OpenShift install path is experimental. It currently requires running sandbox pods under the `privileged` SCC and installing the gateway with TLS disabled. Use only for evaluation on a private network. +Verify that the selected OpenShift runtime profile permits an unprivileged +process to install a nested seccomp user-notification filter and use Landlock. +OpenShell fails sandbox startup when either capability-free runtime probe fails. -OpenShift's [Security Context Constraints](https://docs.openshift.com/container-platform/latest/authentication/managing-security-context-constraints.html) reject the chart's default pod security settings. Installing on OpenShift requires precreating the namespace, granting the `privileged` SCC to the sandbox service account, and overriding a few chart values so the cluster admission controller can assign UIDs and FS groups itself. - -OpenShell installs sandbox nftables rules as individual commands. On OpenShift -nodes where optional conntrack or packet log expressions are unavailable, those -optional rules can fail without rolling back the required proxy bypass reject -rules. - ## Prerequisites -- OpenShift 4.x cluster with `oc` configured -- Helm 3.x -- [Agent Sandbox](/kubernetes/setup#install-agent-sandbox) controller and CRDs installed - -## Install - - +- OpenShift 4.x cluster with `oc` configured. +- Helm 3.x. +- [Agent Sandbox](/kubernetes/setup#install-agent-sandbox) controller and CRDs. +- A CNI that enforces ingress and egress `NetworkPolicy` in sandbox namespaces. -## Create the namespace +## Install OpenShell -Pre-create the namespace so the SCC binding can be applied before the chart installs: +Pre-create the namespace, then install the chart. Keep the default restricted +security posture and acknowledge NetworkPolicy only after validating the CNI. ```shell oc create ns openshell -``` - -## Grant the privileged SCC to sandbox pods - -Sandbox pods run under the `openshell-sandbox` service account in the `openshell` namespace and require the `privileged` SCC: - -```shell -oc adm policy add-scc-to-user privileged -z openshell-sandbox -n openshell -``` - -## Install the chart with OpenShift overrides - -```shell helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set server.disableTls=true \ - --set podSecurityContext.fsGroup=null \ - --set securityContext.runAsUser=null + --set supervisor.sandboxRuntime.networkPolicyEnforced=true ``` -| Override | Reason | -|---|---| -| `server.disableTls=true` | Runs the gateway over plaintext HTTP for simpler evaluation. | -| `podSecurityContext.fsGroup=null` / `securityContext.runAsUser=null` | Clear the chart's hardcoded UID and fsGroup so OpenShift's SCC admission can assign them. | +The driver reads the namespace's `openshift.io/sa.scc.uid-range` annotation and +uses the resulting UID/GID for the sandbox, agent, trusted init containers, and +supervisor. Each container sets `allowPrivilegeEscalation: false`, drops all +Linux capabilities, and uses `RuntimeDefault` seccomp. -## Wait for the gateway to be ready +Wait for the gateway: ```shell oc -n openshell rollout status statefulset/openshell ``` -If you set `workload.kind=deployment`, use -`oc -n openshell rollout status deployment/openshell` instead. - - +If you set `workload.kind=deployment`, wait for `deployment/openshell` instead. -## Connect to the gateway +## Connect to the Gateway -The gateway is now running over plaintext HTTP. Connect with `oc port-forward`: +Forward the gateway port for local evaluation: ```shell oc -n openshell port-forward svc/openshell 8080:8080 -``` - -Register the gateway with the CLI: - -```shell -openshell gateway add http://127.0.0.1:8080 --local --name openshift +openshell gateway add https://127.0.0.1:8080 --local --name openshift openshell status ``` -## Production: expose externally with a real certificate - -The steps above run the gateway over plaintext HTTP for quick evaluation. For -a real deployment, cert-manager can issue the gateway's server certificate -from a real Issuer or ClusterIssuer (for example, an ACME issuer), and an -OpenShift Route with TLS passthrough exposes it externally while the gateway -keeps terminating its own TLS and mTLS. - -Install cert-manager and configure a working `ClusterIssuer` first — see -[Managing Certificates](/kubernetes/managing-certificates) for the -`certManager.serverIssuerRef` details. Configure an OIDC provider as described -in [Access Control](/kubernetes/access-control) — remote gateways authenticate -CLI users via OIDC, not mTLS, so the gateway must know the OIDC issuer URL. -Install the chart with: - -```shell -helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ - --version \ - --namespace openshell \ - --set podSecurityContext.fsGroup=null \ - --set securityContext.runAsUser=null \ - --set server.disableTls=false \ - --set certManager.enabled=true \ - --set certManager.serverIssuerRef.name= \ - --set certManager.serverIssuerRef.kind=ClusterIssuer \ - --set certManager.serverDnsNames[0]= \ - --set openshiftRoute.enabled=true \ - --set openshiftRoute.host= \ - --set server.oidc.issuer= \ - --set server.oidc.audience= -``` - -| Override | Reason | -|---|---| -| `certManager.serverIssuerRef` | Creates a second server certificate from your Issuer or ClusterIssuer for external clients. The gateway uses SNI to present this cert for the external hostname while continuing to present the internal (chart CA) cert to supervisors. The internal certificate's `ca.crt` is the chart CA that also signed the client cert, so the default `clientCaFromServerTlsSecret=true` is correct. | -| `openshiftRoute.enabled` / `openshiftRoute.host` | Creates an OpenShift Route with TLS passthrough — the router forwards the encrypted connection by SNI without decrypting, so the gateway uses the SNI hostname to select the external certificate. | -| `server.oidc.issuer` / `server.oidc.audience` | Configures server-side OIDC validation. Without these, the gateway expects mTLS client certificates and rejects OIDC-only CLI connections. See [Access Control](/kubernetes/access-control). | - -Register the gateway with the CLI over OIDC. Remote gateways authenticate CLI -users via OIDC, not mTLS — see [Access Control](/kubernetes/access-control): - -```shell -openshell gateway add https:// \ - --name openshift \ - --oidc-issuer -openshell gateway login openshift -``` - -## Next Steps - -- For more on certificate provisioning modes, refer to [Managing Certificates](/kubernetes/managing-certificates). -- To expose the gateway externally through the Kubernetes Gateway API instead of a Route, refer to [Ingress](/kubernetes/ingress). -- To configure OIDC authentication, refer to [Access Control](/kubernetes/access-control). +For a remote deployment, configure an OpenShift Route with TLS passthrough and +an OIDC provider. Refer to [Ingress](/kubernetes/ingress), [Managing +Certificates](/kubernetes/managing-certificates), and [Access +Control](/kubernetes/access-control). diff --git a/docs/kubernetes/sandbox-runtime.mdx b/docs/kubernetes/sandbox-runtime.mdx new file mode 100644 index 0000000000..319619ee7f --- /dev/null +++ b/docs/kubernetes/sandbox-runtime.mdx @@ -0,0 +1,128 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "Kubernetes Sandbox Runtime" +sidebar-title: "Sandbox runtime" +description: "Understand how Kubernetes places and protects the sandbox runtime and supervisor." +keywords: "Generative AI, Cybersecurity, Kubernetes, Sandboxing, Network Policy, Seccomp, Landlock" +position: 2 +--- + +OpenShell runs each Kubernetes sandbox as two separately scheduled workloads. +The sandbox owns the agent process. The supervisor owns gateway credentials, +policy decisions, and upstream connections. + +## Understand the Components + +The Kubernetes driver always uses this placement: + +```mermaid +flowchart LR + Gateway[OpenShell gateway] + Supervisor[Supervisor Pod] + Sandbox[Sandbox workload Pod] + Agent[Agent processes] + External[External services] + + Gateway <-->|JWT-authenticated session| Supervisor + Supervisor <-->|TLS and bootstrap token| Sandbox + Sandbox --> Agent + Supervisor --> External +``` + +`openshell-sandbox` runs as PID 1 in the workload container. It launches the +agent, applies Landlock and child seccomp filters, identifies the process behind +each network operation, and relays approved streams. `openshell-supervisor` runs +in a directly managed Pod. It authenticates to the gateway, evaluates policy, +handles L7 and provider transformations, and opens upstream connections. + +Both containers run as the same namespace-resolved non-root UID and GID. Their +Pod specs set `allowPrivilegeEscalation: false`, drop every Linux capability, +and use `RuntimeDefault` seccomp. The sandbox installs an additional nested +seccomp user-notification filter without requesting a capability. Startup fails +closed if the runtime blocks the required seccomp or Landlock operations. + +The supervisor Pod points directly to the Sandbox resource with a non-controller +owner reference. Kubernetes therefore removes it when the Sandbox is deleted, +while the Agent Sandbox controller remains the sole controller of the workload +Pod. + +## Enforce Network Isolation + +The driver creates one workload fence per sandbox namespace before it releases +any workload Pod: + +```yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +spec: + podSelector: + matchLabels: + openshell.ai/boundary-role: workload + policyTypes: [Ingress, Egress] + ingress: + - from: + - podSelector: + matchLabels: + openshell.ai/boundary-role: supervisor + ports: + - protocol: TCP + port: 5500 + egress: [] +``` + +The empty egress list blocks direct DNS, gateway, node, metadata, and Internet +connections from every OpenShell workload in the namespace. The ingress rule +allows OpenShell supervisor Pods to reach sandbox TLS listeners. TLS, JWT +claims, session generation, and recorded Pod UIDs—not the NetworkPolicy—bind a +supervisor to its exact sandbox. Supervisors can reach cluster DNS, the gateway, +and policy-approved upstream destinations unless another namespace policy +restricts them. + +Kubernetes policies are additive. Keep sandbox namespaces under administrative +control so another principal cannot add permissive policies, create Pods with +OpenShell labels, or read bootstrap Secrets. Set +`supervisor.sandboxRuntime.networkPolicyEnforced: true` only after you verify that the +cluster CNI enforces both ingress and egress policies for these namespaces. + +## Bootstrap a Sandbox + +The driver creates each sandbox generation in a fail-closed order: + +1. Create and validate the workload egress fence. +2. Create the Sandbox resource with a scheduling gate. +3. Inspect the admitted Pod identity, security context, DNS settings, and + generation-specific Secret reference. +4. Create a gated supervisor Pod and separate immutable Secrets for sandbox and + supervisor trust material. +5. Remove both scheduling gates. +6. Publish readiness only after the supervisor attaches, confirms enforcement, + and registers the gateway relay. + +The trusted sandbox init container copies its Secret into a memory-backed +volume. The main container never mounts the projected Secret and removes the +staged bootstrap before it launches untrusted code. The supervisor receives an +audience-bound Kubernetes token, exchanges it for a sandbox-scoped JWT, and +keeps gateway and provider credentials outside the workload Pod. + +Stopping a sandbox removes both the workload and supervisor Pods. Starting it +creates a new generation with new Secrets and a replacement supervisor Pod +while preserving the workspace PVC. The namespace-wide workload fence remains +in place across sandbox generations. + +## Check Cluster Requirements + +This architecture requires the following cluster behavior: + +- Linux nodes and a container runtime that permits an unprivileged process to + install a nested seccomp user-notification filter under `RuntimeDefault`. +- Landlock enabled and usable by the non-root sandbox process. +- A CNI that enforces `networking.k8s.io/v1` ingress and egress policies, + including node-local and metadata destinations. +- Support for Pod scheduling gates and the safe + `net.ipv4.ip_unprivileged_port_start=0` sysctl. +- Administrative control of sandbox namespaces and OpenShell role labels. + +OpenShell actively probes the Linux primitives and validates the admitted Pod +before starting the agent. Treat a failed probe or changed security posture as +an unsupported runtime, not a degraded mode. diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index fb7881af2d..c3f4e2416e 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -85,7 +85,8 @@ Install from the OCI registry on GHCR. Replace `` with the chart versio helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ - --namespace openshell + --namespace openshell \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true ``` To use the latest development build instead of a stable release: @@ -94,7 +95,8 @@ To use the latest development build instead of a stable release: helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version 0.0.0-dev \ - --namespace openshell + --namespace openshell \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true ``` The chart automatically generates PKI secrets on first install using pre-install Helm hooks. No manual secret creation is required. @@ -109,6 +111,7 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set workspaceResources.enabled=false \ --set server.sandboxNamespace=app-a @@ -199,14 +202,12 @@ The most commonly changed values are: | `server.sandboxImage` | Default sandbox image used when a sandbox does not specify one. | | `server.sandboxImagePullSecrets` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | | `server.grpcEndpoint` | Endpoint that sandbox supervisors use to call back to the gateway. Must be reachable from inside the cluster. | -| `server.appArmorProfile` | AppArmor profile requested for sandbox agent containers. Defaults to `Unconfined`. | | `server.disableTls` | Run the gateway over plaintext HTTP. Use only behind a trusted transport. | | `server.auth.allowUnauthenticatedUsers` | Accept user-facing calls without OIDC or mTLS credentials. Use only for trusted local development or a fully trusted access proxy. | | `server.enableLoopbackServiceHttp` | Enable local plaintext HTTP for loopback sandbox service URLs. Defaults to `true`. | | `pkiInitJob.serverDnsNames` / `certManager.serverDnsNames` | Additional gateway server DNS SANs. Wildcard SANs also enable sandbox service URLs under that domain. | -| `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect based on cluster version: clusters running Kubernetes 1.35 or later use `image-volume` (ImageVolume GA in 1.36); older clusters use `init-container`. Set explicitly to `image-volume` on Kubernetes 1.33 or 1.34 with the ImageVolume feature gate enabled, or to `init-container` to force the legacy path on any version. | -| `supervisor.topology` | Sandbox pod topology. Refer to [Topology](/kubernetes/topology). | -| `supervisor.sidecar.proxyUid` | Non-root UID used when sidecar process/binary-aware network policy is disabled. The default binary-aware sidecar runs as UID 0 instead. The configured UID must not match the sandbox UID. | +| `supervisor.sandboxRuntime.networkPolicyEnforced` | Required acknowledgement that the cluster CNI enforces ingress and egress `NetworkPolicy` in sandbox namespaces. | +| `supervisor.sandboxRuntime.boundaryPort` | Non-privileged TLS port used between paired supervisor and sandbox Pods. | | `upstreamProxy` | Operator-owned corporate HTTP forward proxy for policy-approved TLS egress. Refer to [Configure a Corporate Upstream Proxy](#configure-a-corporate-upstream-proxy). | Use a values file for repeatable deployments: @@ -216,16 +217,10 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --values my-values.yaml ``` -The chart defaults `server.appArmorProfile` to `Unconfined` because -runtime/default AppArmor profiles can block the supervisor's network namespace -mount setup on AppArmor-enabled nodes. Set `server.appArmorProfile` to an empty -string to omit the field, `RuntimeDefault` to force the runtime default, or -`Localhost/` when you load and manage a localhost profile on each -node. - To use private sandbox images, create a `kubernetes.io/dockerconfigjson` Secret in the sandbox namespace and reference its name: @@ -265,13 +260,12 @@ upstreamProxy: key: credentials authAllowInsecure: true -supervisor: - topology: sidecar ``` Use `authAllowInsecure: true` only when you accept that Basic authentication is cleartext on the connection to an `http://` proxy. The initial release supports `http://` proxy endpoints and TLS CONNECT egress. It does not support HTTPS-to-proxy, custom corporate CA bundles, or forwarding plain HTTP egress through the proxy. -Proxy credentials require `sidecar` topology. It mounts the credential only into the dedicated network supervisor container. OpenShell rejects credential Secrets with `combined` topology because Kubernetes `fsGroup` volume permission handling can make a shared credential mount readable by the sandbox group. +The credential mounts only in the separately scheduled supervisor Pod. The +sandbox workload cannot read it through its environment or volumes. ## RBAC @@ -306,6 +300,7 @@ helm upgrade --install openshell \ oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set serviceAccount.create=false \ --set serviceAccount.name=my-existing-sa ``` @@ -321,7 +316,7 @@ The gateway exposes `/healthz` for process liveness and `/readyz` for dependency ## Next Steps -- To choose between combined and sidecar sandbox pods, refer to [Topology](/kubernetes/topology). +- Kubernetes sandboxes use separate workload and directly managed supervisor Pods; refer to [Sandbox runtime](/kubernetes/sandbox-runtime). - To enable automatic certificate rotation with cert-manager, refer to [Managing Certificates](/kubernetes/managing-certificates). - To expose the gateway externally without port-forwarding, refer to [Ingress](/kubernetes/ingress). - To configure OIDC or reverse-proxy authentication, refer to [Access Control](/kubernetes/access-control). diff --git a/docs/kubernetes/topology.mdx b/docs/kubernetes/topology.mdx deleted file mode 100644 index 869fc07f1b..0000000000 --- a/docs/kubernetes/topology.mdx +++ /dev/null @@ -1,249 +0,0 @@ ---- -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -title: "Kubernetes Sandbox Topology" -sidebar-title: "Topology" -description: "Choose between combined and sidecar supervisor topology for Kubernetes sandbox pods." -keywords: "Generative AI, Cybersecurity, Kubernetes, Sandboxing, Sidecar, Network Policy, RuntimeClass" -position: 2 ---- - -Kubernetes sandbox pods can run the OpenShell supervisor in `combined` or -`sidecar` topology. Choose the topology based on which controls you need inside -the pod and how much privilege your cluster allows on the agent container. - -## Choose a Topology - -The default `combined` topology preserves the full OpenShell enforcement model. -Use `sidecar` only when you accept network-focused enforcement in exchange for a -lower-privilege agent container. - -| Topology | Use when | Main tradeoff | -|---|---|---| -| `combined` | You need OpenShell network, filesystem, and process controls in the sandbox workload. | The agent container carries the Linux capabilities the supervisor needs. | -| `sidecar` | You need the agent container to run as non-root without added Linux capabilities, and network policy is the primary control. | Privilege-dropping and supervisor mount isolation do not run in the agent container. | - -## Privilege Model - -The long-running container permissions differ by topology: - -| Topology | Pod or container | UID/GID | Privilege escalation | Capabilities | Result | -|---|---|---|---|---|---| -| `combined` | Agent container, which also runs the supervisor | Not forced by topology | Not explicitly disabled by the driver | Adds `SYS_ADMIN`, `NET_ADMIN`, `SYS_PTRACE`, and `SYSLOG`; adds `SETUID`, `SETGID`, and `DAC_READ_SEARCH` when user namespaces are enabled | Full supervisor controls run in the agent container. | -| `sidecar` | Agent container, process-only supervisor (`network-only`) | `sandbox_uid:sandbox_gid` | `false` | Drops `ALL` | Agent and workload run without added Linux capabilities. | -| `sidecar` | Network supervisor sidecar, binary-aware mode (default) | `0:sandbox_gid` | `false` | Drops `ALL`; adds `SYS_PTRACE` and `DAC_READ_SEARCH` | Root sidecar inspects cross-UID workload `/proc` entries. The nftables fence exempts UID 0, so do not inject other root containers into these pods. | -| `sidecar` | Network supervisor sidecar, endpoint/L7-only mode | `proxyUid:sandbox_gid` | `false` | Drops `ALL` | Non-root sidecar enforces endpoint and L7 policy without matching `policy.binaries`. | - -Short-lived setup containers still have the permissions needed to prepare the -pod: - -| Topology | Setup container | UID/GID | Privilege escalation | Capabilities | Purpose | -|---|---|---|---|---|---| -| `combined` | Supervisor install init container | `0` | Not set | Not set | Copies the supervisor binary into the agent container volume. | -| `sidecar` | Network init container | `0` | `false` | Drops `ALL`; adds `NET_ADMIN`, `NET_RAW`, `CHOWN`, and `FOWNER` | Installs pod-local nftables rules and prepares shared sidecar state. | - -## Combined Topology - -Combined topology is the original Kubernetes mode and remains the default. The -agent container starts the OpenShell supervisor, and the supervisor launches the -workload after applying sandbox setup. - -```mermaid -flowchart TB - Sandbox["agents.x-k8s.io Sandbox"] - - subgraph Pod["Sandbox pod"] - subgraph Agent["agent container"] - Supervisor["OpenShell supervisor
network + process + filesystem"] - Workload["Agent workload"] - end - end - - Gateway["OpenShell Gateway"] - External["External services"] - - Sandbox --> Pod - Supervisor --> Workload - Supervisor -->|"gateway callback / SSH relay"| Gateway - Supervisor -->|"policy-enforced egress"| External -``` - -Combined topology keeps these controls in one supervisor path: - -- Network endpoint and L7 policy enforcement. -- Filesystem policy enforcement. -- Process and binary identity checks. -- Privilege drop into the sandbox user. -- Gateway relay, SSH sessions, exec, and file sync. - -Because the supervisor performs network namespace setup and process/filesystem -controls from the agent container, Kubernetes grants that container elevated -Linux capabilities. Use this mode when you need the complete OpenShell sandbox -contract and your cluster policy permits those capabilities. - -## Sidecar Topology - -Sidecar topology splits the supervisor into a network sidecar and a -low-privilege process supervisor in the agent container. - -```mermaid -flowchart TB - Sandbox["agents.x-k8s.io Sandbox"] - - subgraph Pod["Sandbox pod"] - Init["network init container
root setup capabilities"] - State["shared state + TLS volumes"] - NetNS["pod network namespace"] - - subgraph Agent["agent container"] - ProcessSupervisor["process supervisor
network-only"] - Workload["Agent workload"] - end - - NetworkSidecar["network supervisor sidecar
UID 0 by default"] - SshEndpoint["abstract SSH relay socket
peer-PID authenticated"] - end - - Gateway["OpenShell Gateway"] - External["External services"] - - Sandbox --> Pod - Init -->|"installs nftables rules"| NetNS - ProcessSupervisor --> Workload - Workload -->|"egress redirected on loopback"| NetworkSidecar - NetworkSidecar -->|"gateway session + relays"| Gateway - NetworkSidecar -->|"policy-enforced egress"| External - NetworkSidecar -->|"control socket + proxy TLS"| State - ProcessSupervisor -->|"bootstrap + updates"| State - ProcessSupervisor --> SshEndpoint - NetworkSidecar -->|"SSH relay"| SshEndpoint - NetworkSidecar --- State -``` - -The pod contains these OpenShell-managed pieces: - -| Component | Runs as | Purpose | -|---|---|---| -| Network init container | Root with setup capabilities | Installs pod-level nftables rules and prepares shared sidecar state. | -| Network sidecar | UID 0 by default; `supervisor.sidecar.proxyUid` when binary-aware policy is disabled | Runs the proxy, enforces network policy, owns gateway authentication and the gateway session, and serves local policy/provider state over the sidecar control socket. | -| Agent container | Resolved sandbox UID/GID | Runs the process supervisor and launches the user workload. | - -In this topology, the agent container defaults to `runAsNonRoot: true`, -`allowPrivilegeEscalation: false`, and `capabilities.drop: ["ALL"]`. The -default binary-aware network sidecar runs as UID 0, drops default Linux -capabilities, and adds `SYS_PTRACE` plus `DAC_READ_SEARCH` for cross-UID workload -process identity resolution. Setting -`supervisor.sidecar.processBinaryAwareNetworkPolicy=false` runs the sidecar as -the configured non-root `proxyUid`, omits both capabilities, and downgrades -network policy to endpoint/L7 enforcement without binary matching. The root -init container keeps the setup capabilities needed to configure pod networking. - -Sidecar mode preserves gateway session behavior, including SSH connectivity, -because the network sidecar owns the gateway session and bridges relay requests -to a Linux abstract SSH socket owned by the process supervisor. The relay -verifies the socket peer PID against the authenticated control connection, so -the workload cannot replace the relay endpoint. The agent container does not get a -gateway endpoint, gateway TLS material, or the sandbox bootstrap token in the -default sidecar path. - - -Sidecar mode runs the process supervisor in `network-only` mode. OpenShell still -enforces network endpoint and L7 policy through the sidecar, and the process -supervisor applies Landlock filesystem policy and child seccomp filters where -the kernel/runtime supports them. The process supervisor does not perform -root-to-sandbox privilege dropping because Kubernetes starts the container as -the sandbox UID/GID, and it does not perform supervisor identity mount -isolation because gateway credentials are not mounted into the agent container. -Sidecar pods use `shareProcessNamespace: true` so the network sidecar can -resolve workload process and binary identity through `/proc/`. - - -## Credential Exposure - -Sidecar topology keeps gateway credentials in the network sidecar. The agent -container does not mount the projected ServiceAccount token used for sandbox -token bootstrap, does not mount the sandbox client TLS secret, and does not get -gateway callback environment variables. - -The network sidecar serves the policy and workload-facing provider environment -over a Unix control socket in the shared sidecar state volume. Before launching -the workload, the process supervisor establishes the only accepted connection. -The sidecar validates its UID, GID, and PID with peer credentials, unlinks the -listener, derives the SSH target from trusted configuration, and rejects later -clients. The connection receives bootstrap state and provider-environment -updates after settings polls. If it closes, the network sidecar exits so -Kubernetes recreates the one-client bootstrap listener, and the process -supervisor exits so Kubernetes terminates the workload and restarts the agent -container. This symmetric failure behavior prevents a surviving workload from -claiming the new control listener after an isolated sidecar restart. Future -child processes can see refreshed provider env without giving the agent -container gateway authentication material. This does not mutate the environment -of the already-running workload entrypoint. Use `combined` topology when you -need the full single-supervisor enforcement path; use additional runtime -isolation when you need a stronger container boundary around sidecar workloads. - -## RuntimeClass Isolation - -Sidecar topology has been validated with Kata Containers. It does not currently -support gVisor because sidecar mode requires pod-local nftables setup, which -gVisor does not provide to the init container. A supported sandboxed runtime -strengthens the container boundary while OpenShell focuses on network policy -enforcement from the sidecar. - -Runtime classes do not re-enable the OpenShell privilege-drop or supervisor -mount-isolation controls that sidecar mode relaxes. Use them as an additional -workload boundary, not as a replacement for the combined topology's full -supervisor controls. - -You can set a default runtime class in the Kubernetes driver configuration or -override it per sandbox with driver config: - -```shell -openshell sandbox create \ - --driver-config-json '{"kubernetes":{"pod":{"runtime_class_name":"kata-containers"}}}' \ - -- claude -``` - -## Enable Sidecar Mode - -For direct gateway TOML configuration, set the Kubernetes driver fields: - -```toml -[openshell.drivers.kubernetes] -topology = "sidecar" - -[openshell.drivers.kubernetes.sidecar] -proxy_uid = 1337 -``` - -`proxy_uid` configures only the relaxed endpoint/L7-only sidecar. It must be at -least `1000` and must not match the sandbox UID. The default binary-aware mode -runs the sidecar as UID 0 instead. The network init container exempts the -effective sidecar UID from proxy redirection so the sidecar can reach the -gateway. - -When the Helm chart renders `gateway.toml`, set the equivalent chart values: - -```yaml -supervisor: - topology: sidecar - sidecar: - proxyUid: 1337 - processBinaryAwareNetworkPolicy: true -``` - -Leave `topology` unset, or set it to `combined`, to keep the original -single-container supervisor path. For Helm installs, leave -`supervisor.topology` unset or set it to `combined`. - -Set `supervisor.sidecar.processBinaryAwareNetworkPolicy=false` only when you -accept downgrading sidecar network policy to endpoint/L7 enforcement without -matching `policy.binaries`. This changes the sidecar from UID 0 to `proxyUid` -and removes its `SYS_PTRACE` and `DAC_READ_SEARCH` capabilities, which are used -for cross-UID `/proc` inspection. - -## Next Steps - -- To install OpenShell on Kubernetes, refer to [Setup](/kubernetes/setup). -- To configure gateway authentication, refer to [Access Control](/kubernetes/access-control). -- To review the driver fields, refer to [Gateway Configuration File](/reference/gateway-config). diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index d36aa0ff51..ed9b38383d 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -94,7 +94,7 @@ future version. To migrate an existing file: omit Docker or Podman `sandbox_pids_limit` to use OpenShell's default limit of 2048, and omit Podman `health_check_interval_secs` to disable health checks. Explicit zero values are invalid. -7. Remove local Docker, Podman, or VM `grpc_endpoint` when the topology-derived +7. Remove local Docker, Podman, or VM `grpc_endpoint` when the driver-derived callback is correct, or retain it as an explicit override. Kubernetes raw TOML requires an explicit endpoint; Helm derives one from the release's gateway Service. New VM root filesystems use an image-provided `sandbox` @@ -546,13 +546,6 @@ image_pull_secrets = ["regcred"] # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" supervisor_image_pull_policy = "if_not_present" -# Use the image volume on Kubernetes >= 1.35 (GA in 1.36); switch to "init-container" -# on older clusters or where the ImageVolume feature gate is off. -supervisor_sideload_method = "image-volume" -# "combined" runs the existing single supervisor container with full process, -# filesystem, and network enforcement in the agent container. "sidecar" moves -# pod-level network enforcement and gateway session handling into a network sidecar. -topology = "combined" # Optional corporate HTTP forward proxy for policy-approved TLS egress. The # sandbox workload cannot select or override these settings. Only http:// proxy # endpoints and TLS CONNECT traffic are supported; plain HTTP egress remains @@ -570,9 +563,7 @@ topology = "combined" # Secret prevents that Pod from starting; unreadable or malformed `user:pass` # content is validated fail-closed by the supervisor at startup and never # falls back to direct egress. -# Proxy credential Secrets require `topology = "sidecar"`. Combined topology -# shares its credential mount with the workload and can make it readable by the -# sandbox group through Kubernetes `fsGroup` volume permission handling. +# Proxy credentials mount only in the separately scheduled supervisor Pod. # Required with a credential Secret: Basic authentication to an http:// proxy # is cleartext on the connection to that proxy. # proxy_auth_allow_insecure = true @@ -587,7 +578,6 @@ ssh_socket_path = "/run/openshell/ssh.sock" client_tls_secret_name = "openshell-client-tls" host_gateway_ip = "10.0.0.1" enable_user_namespaces = false -app_armor_profile = "Unconfined" workspace_default_storage_size = "10Gi" # Kubernetes StorageClass for the workspace PVC. Empty (default) omits the # field, using the cluster's default StorageClass. Set this on clusters with no @@ -622,19 +612,12 @@ enabled = true gateway_namespace = "openshell" gateway_pod_selector = { "app.kubernetes.io/name" = "openshell", "app.kubernetes.io/instance" = "openshell" } -[openshell.drivers.kubernetes.sidecar] -# UID used by relaxed long-running network sidecars. Strict process/binary-aware -# sidecars run as UID 0 so Kubernetes grants the required /proc inspection -# capabilities into the effective set. In sidecar topology the network init -# container installs nftables rules that exempt the effective sidecar UID, so -# this dedicated infrastructure UID must remain at least 1000 and must not -# match the sandbox workload UID. -proxy_uid = 1337 -# Keep process/binary-aware network policy enabled in sidecar topology. Set -# false to run the sidecar as proxy_uid, drop the sidecar's extra /proc -# inspection capabilities, and enforce endpoint/L7 policy without matching -# policy.binaries. -process_binary_aware_network_policy = true +[openshell.drivers.kubernetes.sandbox_runtime] +# Required acknowledgement that the cluster CNI enforces NetworkPolicy and the +# sandbox namespaces prevent untrusted policy, pod, label, and Secret changes. +network_policy_enforced = true +# TLS-protected boundary listener reached only by the paired control pod. +boundary_port = 5500 ``` In managed workspace mode, the Kubernetes driver copies each explicitly named @@ -709,7 +692,7 @@ image_pull_policy = "if_not_present" # Value assigned to the openshell.sandbox_namespace label on sandbox containers. sandbox_label = "docker-dev" # Optional override. When omitted, the gateway derives -# https://host.openshell.internal: for this topology. +# https://host.openshell.internal: for this driver. grpc_endpoint = "https://host.openshell.internal:17670" # The workload runtime and supervisor companion use separate images. Both # default to the gateway version; override either to pin a specific build. @@ -917,7 +900,7 @@ state_dir = "/var/lib/openshell/vm" driver_dir = "/usr/local/libexec/openshell" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" # Optional override. When omitted, the gateway derives -# https://host.openshell.internal: for the VM topology. +# https://host.openshell.internal: for the VM driver. grpc_endpoint = "https://host.openshell.internal:17670" # Empty falls back to default_image. bootstrap_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index c896a07d2a..67a45a3ae1 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -242,7 +242,7 @@ OpenShell rejects mount `source`, `target`, and Docker volume `subpath` values with surrounding whitespace. OpenShell also rejects mount targets that replace the workspace root or container root, or contain or are contained by the configured SSH socket or reserved `/opt/openshell`, `/etc/openshell`, -`/etc/openshell-tls`, `/run/openshell`, `/run/openshell-sidecar`, and network +`/etc/openshell-tls`, `/run/openshell`, and network namespace roots. These checks do not make host bind mounts safe. ## Podman Driver @@ -421,19 +421,16 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `[managed_ssh_ingress]` | `networkPolicy.enabled` | In managed mode, create an SSH ingress policy in every workspace namespace. Helm configures the gateway namespace and pod selector automatically. Operator mode leaves namespace policy management to the platform operator. | | `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. | | `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. | -| `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the supervisor image that provides the `openshell-sandbox` binary. The default repository with an empty tag uses the version-pinned image built into the gateway. Changing the repository uses the effective gateway image tag, while setting a tag pins that version explicitly. | +| `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the trusted runtime image that provides the `openshell-sandbox` and `openshell-supervisor` binaries. The default repository with an empty tag uses the version pinned into the gateway. | | `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | -| `supervisor_sideload_method` | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect from cluster version. Set to `image-volume` to mount the supervisor OCI image directly as a volume (requires Kubernetes 1.33+ with the ImageVolume feature gate; GA in 1.36), or `init-container` to copy it through an init container on older clusters. | -| `topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, or `sidecar` to move pod-level network enforcement and the gateway session into a dedicated sidecar. | +| `sandbox_runtime.network_policy_enforced` | `supervisor.sandboxRuntime.networkPolicyEnforced` | Acknowledge that the cluster CNI enforces ingress and egress `NetworkPolicy` in sandbox namespaces. This must be `true`. | +| `sandbox_runtime.boundary_port` | `supervisor.sandboxRuntime.boundaryPort` | Set the non-privileged TLS port used between the paired supervisor and sandbox Pods. | | `https_proxy` | `upstreamProxy.url` | Set the operator-owned `http://host:port` corporate forward proxy used for policy-approved TLS CONNECT egress. | | `no_proxy` | `upstreamProxy.noProxy` | Set destinations that bypass only the corporate proxy. OpenShell policy evaluation still applies. | -| `proxy_auth_secret_name` | `upstreamProxy.authSecret.name` | Set the existing Secret name in the sandbox namespace that contains the proxy credential. Requires `sidecar` topology. | -| `proxy_auth_secret_key` | `upstreamProxy.authSecret.key` | Set the Secret key containing the `user:pass` credential. Requires `sidecar` topology. | +| `proxy_auth_secret_name` | `upstreamProxy.authSecret.name` | Set the existing Secret name in the sandbox namespace that contains the proxy credential. The Secret mounts only in the supervisor Pod. | +| `proxy_auth_secret_key` | `upstreamProxy.authSecret.key` | Set the Secret key containing the `user:pass` credential. | | `proxy_auth_allow_insecure` | `upstreamProxy.authAllowInsecure` | Set `true` to acknowledge that Basic authentication to an HTTP proxy is cleartext. Required with a proxy credential Secret. | | `proxy_connect_by_hostname` | `upstreamProxy.connectByHostname` | Send hostnames rather than validated IPs in CONNECT requests. Use only when proxy ACLs require hostname targets. | -| `sidecar.proxy_uid` | `supervisor.sidecar.proxyUid` | Dedicated UID of at least `1000` used by the relaxed sidecar when process/binary-aware network policy is disabled. It must not match the workload UID. The default binary-aware sidecar runs as UID 0. The network init container exempts the effective sidecar UID from proxy redirection. | -| `sidecar.process_binary_aware_network_policy` | `supervisor.sidecar.processBinaryAwareNetworkPolicy` | Keep process/binary-aware network policy enabled in `sidecar` topology. The default runs the sidecar as UID 0 with `SYS_PTRACE` and `DAC_READ_SEARCH`. Set false to run as `proxy_uid`, drop both capabilities, and enforce endpoint/L7 policy without matching `policy.binaries`. | -| `app_armor_profile` | `server.appArmorProfile` | Set the sandbox agent container's AppArmor profile. Helm defaults this to `Unconfined` so AppArmor-enabled nodes do not block supervisor network namespace setup. Set the Helm value to an empty string to omit the field, or use `RuntimeDefault` or `Localhost/` for operator-managed profiles. | | `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Set the default workspace PVC size for new sandboxes. | | `workspace_storage_class` | `server.workspaceStorageClass` | Set the `StorageClass` for the workspace PVC. Empty (default) omits `storageClassName` and uses the cluster's default `StorageClass`. Set this on clusters with no default `StorageClass`, otherwise the workspace PVC stays `Pending` and the sandbox never starts. | | `sa_token_ttl_secs` | `server.sandboxJwt.k8sSaTokenTtlSecs` | Set the projected ServiceAccount token TTL used for the bootstrap token exchange. | @@ -445,30 +442,17 @@ remain limited to the explicitly configured TLS and image-pull Secret names. The driver creates copies only in gateway-owned managed namespaces. Do not reuse the gateway ServiceAccount for unrelated workloads. -In `combined` topology, the agent container carries the Linux capabilities -needed by the supervisor for network namespace setup, Landlock filesystem -policy, process privilege changes, and network policy enforcement. In `sidecar` -topology, the agent container runs as the resolved sandbox UID/GID with no added -Linux capabilities. A root init container performs the nftables setup, and the -long-running binary-aware sidecar runs as UID 0, drops default capabilities, -and adds `SYS_PTRACE` plus `DAC_READ_SEARCH` for workload process identity -resolution through shared `/proc`. The -`sidecar.process_binary_aware_network_policy = false` setting runs it as the -configured non-root `proxy_uid`, removes both capabilities, and relaxes network -policy to endpoint/L7 matching only. The -network sidecar owns gateway authentication and writes local policy/provider -state to the process supervisor over a local control socket, so the agent -container does not mount the sandbox bootstrap token or client TLS secret in -the default sidecar path. The provider environment is refreshed by the network -sidecar after settings polls and streamed to the process supervisor so future -child processes can see updated provider env without gateway access in the -agent container. -Sidecar mode keeps gateway session and SSH behavior. The process supervisor -applies Landlock filesystem policy and child seccomp filters where supported, -but it does not perform root-to-sandbox privilege dropping or supervisor -identity mount isolation. Network policy still runs in the sidecar, and sidecar -pods set `shareProcessNamespace: true` so the network sidecar can resolve -process/binary identity through `/proc/`. +The Kubernetes driver always places the sandbox runtime in the workload Pod and +the supervisor in a separate, directly managed Pod. The +workload Pod runs `openshell-sandbox` as the same non-root UID/GID as the agent +and requests no added Linux capabilities. The supervisor Pod runs +`openshell-supervisor`, authenticates to the gateway with a sandbox JWT, and +owns upstream connections. Both containers disable privilege escalation, drop +all capabilities, and use `RuntimeDefault` seccomp. The sandbox adds a nested +seccomp user-notification filter and Landlock restrictions before it launches +the agent. One namespace-wide, empty-egress `NetworkPolicy` is the mandatory +outer fence for all OpenShell workload Pods. It permits supervisor Pods to +reach sandbox listeners; TLS and JWT identity enforce the exact pairing. The Kubernetes driver creates namespaced `agents.x-k8s.io` `Sandbox` resources from the Kubernetes SIG Apps [agent-sandbox](https://github.com/kubernetes-sigs/agent-sandbox) project. It detects the served Sandbox API at runtime, caches the selected API version for the gateway process, and uses `v1beta1` when available before falling back to `v1alpha1`, so supported Agent Sandbox installations work without version-specific operator configuration. The Agent Sandbox controller turns those resources into sandbox pods and related storage. diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 21c71375a6..f4202319ae 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -90,7 +90,7 @@ to forward conversation text. Malformed placeholder candidates and candidates longer than 4096 wire bytes are denied. Authentication headers and HTTP trailers retain their credential restrictions. -In the separate sandbox/supervisor topology, the supervisor mediates DNS by +When `openshell-sandbox` and the supervisor run separately, the supervisor mediates DNS by hostname. DNS sender identity is unavailable because native socket writes can come from a process that inherited the socket or replaced its executable. Resolving a name does not authorize a connection: the supervisor still checks diff --git a/e2e/docker/Dockerfile.external-kubernetes-gateway b/e2e/docker/Dockerfile.external-kubernetes-gateway index 4bda320440..20738a512a 100644 --- a/e2e/docker/Dockerfile.external-kubernetes-gateway +++ b/e2e/docker/Dockerfile.external-kubernetes-gateway @@ -8,6 +8,7 @@ FROM ${GATEWAY_BASE_IMAGE} ARG TARGETARCH ARG SUPERVISOR_IMAGE=ghcr.io/nvidia/openshell/supervisor:latest +ARG SANDBOX_RUNTIME_IMAGE=ghcr.io/nvidia/openshell/sandbox:latest COPY deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-gateway /usr/local/bin/openshell-gateway COPY deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-driver-kubernetes /usr/local/bin/openshell-driver-kubernetes @@ -21,8 +22,10 @@ ENV OPENSHELL_COMPUTE_DRIVER=kubernetes \ OPENSHELL_GRPC_ENDPOINT=http://openshell.openshell.svc.cluster.local:8080 \ OPENSHELL_SUPERVISOR_IMAGE=${SUPERVISOR_IMAGE} \ OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY=if_not_present \ + OPENSHELL_SANDBOX_RUNTIME_IMAGE=${SANDBOX_RUNTIME_IMAGE} \ + OPENSHELL_SANDBOX_RUNTIME_IMAGE_PULL_POLICY=if_not_present \ OPENSHELL_SUPERVISOR_SIDELOAD_METHOD=init-container \ - OPENSHELL_K8S_TOPOLOGY=combined + OPENSHELL_K8S_SANDBOX_RUNTIME_NETWORK_POLICY_ENFORCED=true USER 1000:1000 EXPOSE 8080 diff --git a/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh b/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh index 4d9d2eeff2..43ee52140a 100755 --- a/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh +++ b/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh @@ -3,7 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 # Helm post-renderer for the external Kubernetes compute-driver smoke test. -# It keeps the test-only sidecar and Unix socket plumbing out of the chart. +# It keeps the test-only external-driver container and Unix socket plumbing out +# of the chart. set -euo pipefail diff --git a/e2e/rust/src/harness/sandbox.rs b/e2e/rust/src/harness/sandbox.rs index e6948acbb6..e0a1f35898 100644 --- a/e2e/rust/src/harness/sandbox.rs +++ b/e2e/rust/src/harness/sandbox.rs @@ -7,6 +7,7 @@ //! is dropped, replacing the `trap cleanup EXIT` pattern from the bash tests. use std::process::Stdio; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -31,6 +32,23 @@ fn extract_sandbox_name(output: &str) -> Option { /// startup. const SANDBOX_READY_TIMEOUT: Duration = Duration::from_secs(600); +static NEXT_SANDBOX_NAME: AtomicU64 = AtomicU64::new(1); + +fn has_explicit_sandbox_name(args: &[&str]) -> bool { + args.iter() + .any(|arg| *arg == "--name" || arg.starts_with("--name=")) +} + +fn add_unique_name_if_missing(command: &mut tokio::process::Command, args: &[&str]) { + if !has_explicit_sandbox_name(args) { + command.arg("--name").arg(format!( + "e2e-{}-{}", + std::process::id(), + NEXT_SANDBOX_NAME.fetch_add(1, Ordering::Relaxed) + )); + } +} + /// RAII guard that deletes a sandbox on drop. /// /// For sandboxes created with `--keep` (long-running background command), the @@ -92,6 +110,7 @@ impl SandboxGuard { let mut cmd = openshell_cmd(); cmd.arg("sandbox").arg("create").arg("--detach"); + add_unique_name_if_missing(&mut cmd, create_args); for arg in create_args { cmd.arg(arg); } @@ -166,10 +185,9 @@ impl SandboxGuard { /// which lets tests control competing and reconnecting clients directly. pub async fn create_detached_main(command: &[&str]) -> Result { let mut cmd = openshell_cmd(); - cmd.arg("sandbox") - .arg("create") - .arg("--detach") - .arg("--") + cmd.arg("sandbox").arg("create").arg("--detach"); + add_unique_name_if_missing(&mut cmd, &[]); + cmd.arg("--") .args(command) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -216,6 +234,7 @@ impl SandboxGuard { ) -> Result { let mut create_cmd = openshell_cmd(); create_cmd.arg("sandbox").arg("create").arg("--detach"); + add_unique_name_if_missing(&mut create_cmd, create_args); for arg in create_args { create_cmd.arg(arg); } @@ -354,6 +373,7 @@ impl SandboxGuard { ) -> Result { let mut cmd = openshell_cmd(); cmd.arg("sandbox").arg("create").arg("--detach"); + add_unique_name_if_missing(&mut cmd, &[]); for (local, dest) in uploads { cmd.arg("--upload").arg(format!("{local}:{dest}")); } @@ -680,3 +700,15 @@ impl Drop for SandboxGuard { }); } } + +#[cfg(test)] +mod tests { + use super::has_explicit_sandbox_name; + + #[test] + fn detects_explicit_sandbox_names() { + assert!(has_explicit_sandbox_name(&["--name", "example"])); + assert!(has_explicit_sandbox_name(&["--name=example"])); + assert!(!has_explicit_sandbox_name(&["--policy", "policy.yaml"])); + } +} diff --git a/e2e/rust/tests/kubernetes_corporate_proxy.rs b/e2e/rust/tests/kubernetes_corporate_proxy.rs index 0e83a1a799..3fc1c12a30 100644 --- a/e2e/rust/tests/kubernetes_corporate_proxy.rs +++ b/e2e/rust/tests/kubernetes_corporate_proxy.rs @@ -6,7 +6,7 @@ //! Kubernetes wiring coverage for authenticated corporate forward proxies. //! //! The shell wrapper configures the gateway before it starts, creates the -//! credential Secret, and forces sidecar topology. This test starts a proxy +//! credential Secret, and enables the Kubernetes sandbox runtime. This test starts a proxy //! and an HTTPS upstream on the host visible to sandbox pods, then proves that //! a permitted request uses authenticated CONNECT while a policy-denied port //! never reaches the proxy. diff --git a/e2e/rust/tests/sandbox_labels.rs b/e2e/rust/tests/sandbox_labels.rs index 4de545d7bb..90351266c5 100644 --- a/e2e/rust/tests/sandbox_labels.rs +++ b/e2e/rust/tests/sandbox_labels.rs @@ -115,23 +115,26 @@ async fn delete_sandbox(name: &str) { #[tokio::test] #[allow(clippy::too_many_lines)] // end-to-end test exercises full label lifecycle async fn sandbox_labels_are_stored_and_filterable() { + // Keep the unique names below the public 19-character routable-name + // limit. Hex keeps the per-process suffix compact on busy CI hosts. + let suffix = format!("{:x}", std::process::id()); + let dev_backend = format!("lbl-db-{suffix}"); + let staging_backend = format!("lbl-sb-{suffix}"); + let prod_frontend = format!("lbl-pf-{suffix}"); + let dev_data = format!("lbl-dd-{suffix}"); + // Create sandboxes with different labels let name1 = - create_sandbox_with_labels("e2e-lbl-dev-back", &[("env", "dev"), ("team", "backend")]) - .await; + create_sandbox_with_labels(&dev_backend, &[("env", "dev"), ("team", "backend")]).await; - let name2 = create_sandbox_with_labels( - "e2e-lbl-stg-back", - &[("env", "staging"), ("team", "backend")], - ) - .await; + let name2 = + create_sandbox_with_labels(&staging_backend, &[("env", "staging"), ("team", "backend")]) + .await; let name3 = - create_sandbox_with_labels("e2e-lbl-prd-frnt", &[("env", "prod"), ("team", "frontend")]) - .await; + create_sandbox_with_labels(&prod_frontend, &[("env", "prod"), ("team", "frontend")]).await; - let name4 = - create_sandbox_with_labels("e2e-lbl-dev-data", &[("env", "dev"), ("team", "data")]).await; + let name4 = create_sandbox_with_labels(&dev_data, &[("env", "dev"), ("team", "data")]).await; // Test 1: Verify labels are stored in sandbox metadata let details = get_sandbox_details(&name1).await; diff --git a/e2e/rust/tests/user_namespaces.rs b/e2e/rust/tests/user_namespaces.rs index 53f2c6ec1f..abe29fbc9f 100644 --- a/e2e/rust/tests/user_namespaces.rs +++ b/e2e/rust/tests/user_namespaces.rs @@ -8,8 +8,8 @@ //! Enables `OPENSHELL_ENABLE_USER_NAMESPACES` on the gateway, triggers sandbox //! creation, and inspects the resulting pod spec to confirm: //! 1. `spec.hostUsers` is `false` -//! 2. The container security context includes the extra capabilities -//! (SETUID, SETGID, DAC_READ_SEARCH) required for user namespace operation +//! 2. The container security context requests no added capabilities and +//! drops every capability //! //! The sandbox pod may fail to start in Docker-in-Docker dev clusters where the //! filesystem does not support ID-mapped mounts. The test inspects the pod spec @@ -211,7 +211,7 @@ async fn sandbox_pod_spec_has_user_namespace_fields() { .await; // Inspect capabilities on the agent container. - let caps = kubectl(&[ + let cap_add = kubectl(&[ "get", "pod", &sandbox_name, @@ -221,6 +221,16 @@ async fn sandbox_pod_spec_has_user_namespace_fields() { "jsonpath={.spec.containers[?(@.name=='agent')].securityContext.capabilities.add}", ]) .await; + let cap_drop = kubectl(&[ + "get", + "pod", + &sandbox_name, + "-n", + "openshell", + "-o", + "jsonpath={.spec.containers[?(@.name=='agent')].securityContext.capabilities.drop}", + ]) + .await; // Clean up. stop_child(&mut child).await; @@ -235,18 +245,14 @@ async fn sandbox_pod_spec_has_user_namespace_fields() { "sandbox pod must have spec.hostUsers=false when user namespaces are enabled" ); - // Assert extra capabilities are present. - let caps_val = caps.expect("failed to get capabilities from pod spec"); - for cap in ["SETUID", "SETGID", "DAC_READ_SEARCH"] { - assert!( - caps_val.contains(cap), - "sandbox pod must include {cap} in capabilities when user namespaces are enabled, got: {caps_val}" - ); - } - for cap in ["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE", "SYSLOG"] { - assert!( - caps_val.contains(cap), - "sandbox pod must include {cap} in capabilities, got: {caps_val}" - ); - } + let cap_add = cap_add.expect("failed to get added capabilities from pod spec"); + assert!( + cap_add.trim().is_empty(), + "sandbox pod must not request added capabilities, got: {cap_add}" + ); + let cap_drop = cap_drop.expect("failed to get dropped capabilities from pod spec"); + assert!( + cap_drop.contains("ALL"), + "sandbox pod must drop every capability, got: {cap_drop}" + ); } diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 8a9eaf3df2..8eff5f7787 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -258,9 +258,34 @@ cleanup() { && kctl get namespace "${NAMESPACE}" >/dev/null 2>&1; then echo "=== gateway pod state (preserved for debugging) ===" kctl -n "${NAMESPACE}" get pods -o wide 2>&1 || true + echo "=== Agent Sandbox resources ===" + kctl -n "${NAMESPACE}" get sandboxes.agents.x-k8s.io -o yaml 2>&1 || true + echo "=== gateway sandbox records ===" + "${OPENSHELL_BIN:-${ROOT}/target/debug/openshell}" \ + sandbox list --all-workspaces --output json 2>&1 || true + echo "=== sandbox-runtime supervisor Pods ===" + kctl -n "${NAMESPACE}" get pods \ + -l "openshell.ai/boundary-role=supervisor" -o yaml 2>&1 || true + echo "=== sandbox-runtime supervisor logs (last 200 lines each) ===" + while IFS= read -r supervisor_pod; do + [ -n "${supervisor_pod}" ] || continue + echo "--- ${supervisor_pod} ---" + kctl -n "${NAMESPACE}" logs "${supervisor_pod}" \ + --all-containers --prefix --tail=200 2>&1 || true + echo "--- ${supervisor_pod} (previous containers) ---" + kctl -n "${NAMESPACE}" logs "${supervisor_pod}" --previous \ + --all-containers --prefix --tail=200 2>&1 || true + done < <(kctl -n "${NAMESPACE}" get pods \ + -l "openshell.ai/boundary-role=supervisor" -o name 2>/dev/null || true) echo "=== gateway events ===" kctl -n "${NAMESPACE}" get events --sort-by=.lastTimestamp 2>&1 \ | tail -n 80 || true + echo "=== gateway lifecycle and supervisor-session logs ===" + kctl -n "${NAMESPACE}" logs "$(kube_workload_ref "${RELEASE_NAME}")" \ + --since=20m \ + --all-containers --prefix 2>&1 \ + | grep -Ei "sandbox phase changed|start_sandbox|stop_sandbox|supervisor session|sandbox-runtime|bootstrap" \ + || true echo "=== gateway logs (last 200 lines) ===" kctl -n "${NAMESPACE}" logs \ -l "app.kubernetes.io/instance=${RELEASE_NAME}" --tail=200 \ @@ -417,6 +442,8 @@ run_scenario() { --set "fullnameOverride=openshell" \ --set "image.repository=${REGISTRY_VALUE}/gateway" \ --set "image.tag=${IMAGE_TAG_VALUE}" \ + --set "sandboxRuntime.image.repository=${REGISTRY_VALUE}/sandbox" \ + --set "sandboxRuntime.image.tag=${IMAGE_TAG_VALUE}" \ --set "supervisor.image.repository=${REGISTRY_VALUE}/supervisor" \ --set "supervisor.image.tag=${IMAGE_TAG_VALUE}" \ "${helm_post_renderer_args[@]}" \ @@ -685,6 +712,7 @@ if [ -z "${OPENSHELL_E2E_KUBE_BUILD_IMAGES+x}" ]; then fi fi +reuse_sandbox_image=0 reuse_supervisor_image=0 if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then REGISTRY_VALUE="${OPENSHELL_REGISTRY:-openshell}" @@ -816,6 +844,7 @@ if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then docker build \ --build-arg "TARGETARCH=${external_arch}" \ --build-arg "SUPERVISOR_IMAGE=${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}" \ + --build-arg "SANDBOX_RUNTIME_IMAGE=${REGISTRY_VALUE}/sandbox:${IMAGE_TAG_VALUE}" \ --tag "${REGISTRY_VALUE}/gateway:${IMAGE_TAG_VALUE}" \ --file "${ROOT}/e2e/docker/Dockerfile.external-kubernetes-gateway" \ "${ROOT}" @@ -823,7 +852,16 @@ if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then CONTAINER_ENGINE=docker IMAGE_REGISTRY="${REGISTRY_VALUE}" IMAGE_TAG="${IMAGE_TAG_VALUE}" \ bash "${ROOT}/tasks/scripts/docker-build-image.sh" gateway fi + sandbox_image="${REGISTRY_VALUE}/sandbox:${IMAGE_TAG_VALUE}" supervisor_image="${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}" + if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" != "1" ] \ + || ! docker image inspect "${sandbox_image}" >/dev/null 2>&1; then + CONTAINER_ENGINE=docker IMAGE_REGISTRY="${REGISTRY_VALUE}" IMAGE_TAG="${IMAGE_TAG_VALUE}" \ + bash "${ROOT}/tasks/scripts/docker-build-image.sh" sandbox + else + reuse_sandbox_image=1 + echo "Reusing existing sandbox image ${sandbox_image}" + fi if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" != "1" ] \ || ! docker image inspect "${supervisor_image}" >/dev/null 2>&1; then CONTAINER_ENGINE=docker IMAGE_REGISTRY="${REGISTRY_VALUE}" IMAGE_TAG="${IMAGE_TAG_VALUE}" \ @@ -837,6 +875,7 @@ fi if [ -n "${import_cluster_name}" ]; then for image in \ "${REGISTRY_VALUE}/gateway:${IMAGE_TAG_VALUE}" \ + "${REGISTRY_VALUE}/sandbox:${IMAGE_TAG_VALUE}" \ "${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}"; do if docker image inspect "${image}" >/dev/null 2>&1; then echo "Importing ${image} into k3d cluster ${import_cluster_name}..." @@ -849,6 +888,11 @@ elif [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ] \ && command -v kind >/dev/null 2>&1; then kind_cluster_name="${KUBE_CONTEXT#kind-}" kind_images=("${REGISTRY_VALUE}/gateway:${IMAGE_TAG_VALUE}") + # The CI workflow loads its published sandbox archive before invoking this + # wrapper. Only load a sandbox image here when this script rebuilt it. + if [ "${reuse_sandbox_image}" != "1" ]; then + kind_images+=("${REGISTRY_VALUE}/sandbox:${IMAGE_TAG_VALUE}") + fi # The CI workflow loads its published supervisor archive before invoking this # wrapper. Only load a supervisor image here when this script rebuilt it. if [ "${reuse_supervisor_image}" != "1" ]; then @@ -1038,6 +1082,8 @@ else --set "fullnameOverride=openshell" \ --set "image.repository=${REGISTRY_VALUE}/gateway" \ --set "image.tag=${IMAGE_TAG_VALUE}" \ + --set "sandboxRuntime.image.repository=${REGISTRY_VALUE}/sandbox" \ + --set "sandboxRuntime.image.tag=${IMAGE_TAG_VALUE}" \ --set "supervisor.image.repository=${REGISTRY_VALUE}/supervisor" \ --set "supervisor.image.tag=${IMAGE_TAG_VALUE}" \ "${helm_extra_args[@]}" \ diff --git a/examples/gateway-deploy-connect.md b/examples/gateway-deploy-connect.md index 37ed37bf2e..2861444d1c 100644 --- a/examples/gateway-deploy-connect.md +++ b/examples/gateway-deploy-connect.md @@ -15,6 +15,7 @@ Install the gateway into a Kubernetes cluster you manage: kubectl create namespace openshell helm upgrade --install openshell deploy/helm/openshell \ --namespace openshell \ + --set supervisor.sandboxRuntime.networkPolicyEnforced=true \ --set server.disableTls=true \ --set service.type=ClusterIP ``` diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index df80a936df..a862db2f2f 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -224,6 +224,8 @@ Common findings: upgrade. - Sandbox runtime image exits before printing `openshell-sandbox --version`: verify the configured image contains a static executable at `/openshell-sandbox`. - A sandbox with explicit `protocol: tcp` endpoints fails before workload readiness: confirm the selected isolation backend advertises TCP mediation, then inspect the sandbox and supervisor logs for protected-channel setup or listener failures. A driver that cannot supply the required outer egress fence and authenticated runtime channel must reject the policy before starting the agent. +- Supervisor runtime validation fails: verify `supervisor_image` contains a static `/openshell-supervisor` executable from the same release as the sandbox runtime. +- The sandbox fails its enforcement probe: inspect the sandbox log for the exact nested seccomp user-notification, task-memory, Landlock, loopback DNS, or socket-injection check that failed. Do not add capabilities or switch to an unconfined seccomp profile; use a runtime whose default profile permits the unprivileged probe. - A GPU sandbox fails because Docker reports no discovered NVIDIA CDI devices: verify `.DiscoveredDevices` contains entries such as `nvidia.com/gpu=all`, verify `/etc/cdi` or `/var/run/cdi` contains a generated NVIDIA spec, and check that `nvidia-cdi-refresh.service` and `nvidia-cdi-refresh.path` from NVIDIA Container Toolkit are enabled and healthy. The service is a one-shot unit, so `inactive (dead)` can be normal after a successful run; use `systemctl status` and `journalctl` to distinguish success from a skipped or failed refresh. Restart `nvidia-cdi-refresh.service` to regenerate missing or stale CDI specs, then restart or reload Docker and re-check `docker info`. During a graceful gateway restart, Docker, Podman, and VM sandboxes with @@ -426,7 +428,7 @@ If `server.providerTokenGrants.spiffe.enabled=true`, the gateway should still render `[openshell.gateway.gateway_jwt]` and mount the `sandbox-jwt` Secret. SPIRE is used by both the gateway and sandbox supervisors for dynamic provider token grants. The gateway pod must mount the `spiffe-workload-api` CSI volume -and set `OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET`; sandbox pods must +and set `OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET`; supervisor Pods must receive the matching Workload API socket from the Kubernetes driver config. The gateway verifies supervisor JWT-SVIDs from JWT bundles fetched through this Workload API socket, not from the SPIRE OIDC discovery endpoint. @@ -549,24 +551,49 @@ kubectl -n openshell get configmap openshell-config -o jsonpath='{.data.gateway\ kubectl -n get sandbox -o jsonpath='{.spec.template.spec.serviceAccountName}{"\n"}' ``` -Each Kubernetes sandbox uses a workload Pod and a separate supervisor Pod. -The workload Pod runs `openshell-sandbox`, which owns the agent process tree, -identifies binaries, and intercepts mediated requests. The supervisor Pod runs -`openshell-supervisor`, owns gateway authentication and external egress, and -connects to the workload through the per-sandbox boundary Service. NetworkPolicy -allows only the paired supervisor-to-workload channel and denies direct workload -egress. - -The workload must not receive gateway credentials, supervisor TLS private keys, -or upstream proxy credentials. If startup or mediation fails, inspect both Pods, -the boundary Service, the per-sandbox Secrets, and the selecting NetworkPolicies: - -```bash -kubectl -n openshell get configmap openshell-config -o jsonpath='{.data.gateway\.toml}' -kubectl -n get pod,service,secret,networkpolicy -l openshell.ai/sandbox-id= -kubectl -n logs -c agent --tail=200 -kubectl -n logs --tail=200 -``` +The Kubernetes driver creates a sandbox workload Pod and a separate, directly +managed supervisor Pod. Helm must render +`network_policy_enforced = true`. This is an explicit operator assertion that +the cluster CNI enforces Kubernetes NetworkPolicy; the Kubernetes API cannot +attest enforcement. Run sandboxes only in a trusted namespace +where tenants cannot create Pods, copy OpenShell role labels, or read the +bootstrap Secret. + +The workload Pod runs `/openshell-sandbox`. It has no gateway credentials and +no direct egress. One namespace-wide workload NetworkPolicy is created before +the suspended Sandbox resource. It denies all workload egress and allows +supervisor Pods to reach sandbox TLS listeners. The driver then creates a +per-sandbox Service, split immutable bootstrap Secrets, and a gated supervisor +Pod before releasing either Pod. The supervisor Pod runs +`/openshell-supervisor`. Both Pods use the +same resolved non-root identity, request no capabilities, drop `ALL`, disable +privilege escalation, and use `RuntimeDefault` seccomp. The supervisor reaches +the sandbox over per-sandbox TLS with server-certificate verification plus +bootstrap-token client authentication, and owns gateway policy, provider +credentials, DNS, and mediated upstream connections. + +Inspect all driver-managed resources when a Kubernetes sandbox remains Starting +or loses readiness: + +```bash +kubectl -n get sandbox,pod,service,secret -l openshell.ai/sandbox-id= +kubectl -n get networkpolicy openshell-sandbox-workloads -o yaml +kubectl -n describe pod -l openshell.ai/sandbox-id=,openshell.ai/boundary-role=supervisor +kubectl -n logs pod/ --tail=200 +kubectl -n get pod -l openshell.ai/sandbox-id=,openshell.ai/boundary-role=workload -o yaml +kubectl -n get networkpolicy -l openshell.ai/sandbox-id= -o yaml +``` + +Creation and recovery fail closed. A missing Secret leaves both pods inert; a +missing or unobserved workload fence must prevent the driver from releasing the +Sandbox; and readiness requires both Agent Sandbox readiness and an Available +supervisor Pod. Its exec readiness check succeeds only after the +supervisor has attached, confirmed enforcement, started or resumed the +workload, and registered the gateway access plane. Use both Pod logs for +bootstrap errors. An `EPERM` during enforcement setup means the runtime blocked +a required unprivileged seccomp, task-memory, or Landlock operation. Do not add +capabilities, gateway egress, or credentials to the workload Pod as a +workaround. #### Corporate upstream proxy @@ -610,7 +637,7 @@ destination that should be direct is missing from `no_proxy`. Inspect the network supervisor logs for CONNECT and upstream-proxy decisions: ```bash -kubectl -n logs --tail=200 | grep -Ei 'upstream|connect|proxy' +kubectl -n logs pod/ --tail=200 | grep -Ei 'upstream|connect|proxy' ``` ### Step 7: Check VM-Backed Gateways diff --git a/tasks/helm.toml b/tasks/helm.toml index 525e9c2e51..33b3550dff 100644 --- a/tasks/helm.toml +++ b/tasks/helm.toml @@ -39,12 +39,12 @@ run = """ helm dependency build deploy/helm/openshell echo "--- helm lint: defaults ---" echo "values files: deploy/helm/openshell/values.yaml" - helm lint deploy/helm/openshell --set agentSandbox.preflight.enabled=false + helm lint deploy/helm/openshell --set agentSandbox.preflight.enabled=false --set supervisor.sandboxRuntime.networkPolicyEnforced=true for f in deploy/helm/openshell/ci/values-*.yaml; do variant=$(basename "$f" .yaml | sed 's/values-//') echo "--- helm lint: $variant ---" echo "values files: deploy/helm/openshell/values.yaml, $f" - helm lint deploy/helm/openshell -f "$f" --set agentSandbox.preflight.enabled=false + helm lint deploy/helm/openshell -f "$f" --set agentSandbox.preflight.enabled=false --set supervisor.sandboxRuntime.networkPolicyEnforced=true done echo "--- helm lint: workspace defaults ---" helm lint deploy/helm/openshell-workspace @@ -71,46 +71,16 @@ description = "Run skaffold dev for deploy/helm/openshell (iterative deploy)" dir = "deploy/helm/openshell" run = "skaffold dev" -["helm:skaffold:dev:sidecar"] -description = "Run skaffold dev with the Kubernetes supervisor sidecar topology" -dir = "deploy/helm/openshell" -run = "skaffold dev -p sidecar" - -["helm:skaffold:dev:sidecar-mtls"] -description = "Run skaffold dev with the Kubernetes supervisor sidecar topology and TLS/mTLS enabled" -dir = "deploy/helm/openshell" -run = "skaffold dev -p sidecar-mtls" - ["helm:skaffold:run"] description = "Run a one-shot Skaffold deploy and register its local plaintext gateway" dir = "deploy/helm/openshell" run = "skaffold run && ../../../tasks/scripts/helm-k3s-local.sh register" -["helm:skaffold:run:sidecar"] -description = "Run a sidecar Skaffold deploy and register its local plaintext gateway" -dir = "deploy/helm/openshell" -run = "skaffold run -p sidecar && ../../../tasks/scripts/helm-k3s-local.sh register" - -["helm:skaffold:run:sidecar-mtls"] -description = "Run skaffold run with the Kubernetes supervisor sidecar topology and TLS/mTLS enabled" -dir = "deploy/helm/openshell" -run = "skaffold run -p sidecar-mtls" - ["helm:skaffold:delete"] description = "Run skaffold delete for deploy/helm/openshell" dir = "deploy/helm/openshell" run = "skaffold delete" -["helm:skaffold:delete:sidecar"] -description = "Run skaffold delete for the Kubernetes supervisor sidecar topology" -dir = "deploy/helm/openshell" -run = "skaffold delete -p sidecar" - -["helm:skaffold:delete:sidecar-mtls"] -description = "Run skaffold delete for the Kubernetes supervisor sidecar topology with TLS/mTLS enabled" -dir = "deploy/helm/openshell" -run = "skaffold delete -p sidecar-mtls" - ["helm:skaffold:diagnose"] description = "Run skaffold diagnose for deploy/helm/openshell" dir = "deploy/helm/openshell" diff --git a/tasks/scripts/trivy-scan-test.sh b/tasks/scripts/trivy-scan-test.sh index 8ce49e544b..764a8d1a62 100755 --- a/tasks/scripts/trivy-scan-test.sh +++ b/tasks/scripts/trivy-scan-test.sh @@ -212,10 +212,10 @@ expect_status 0 "scan selected profiles and retain distinct packaged chart versi --chart-ref oci://registry-a.example/charts/helm-chart:1.1.0 \ --chart-ref oci://registry-b.example/charts/helm-chart:1.0.0 jq -se ' - length == 21 + length == 20 and ([.[] | select(.[-1] == "deploy")] | length) == 1 and ([.[] | select(.[-1] == "deploy/helm")] | length) == 1 - and ([.[] | select(.[-1] == "deploy/helm/openshell")] | length) == 16 + and ([.[] | select(.[-1] == "deploy/helm/openshell")] | length) == 15 and all(.[]; (join(" ") | contains("values-spire-stack.yaml")) | not) and all(.[]; index("UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL") != null) ' "${TRIVY_TEST_CALLS}" >/dev/null diff --git a/tasks/scripts/trivy-scan.sh b/tasks/scripts/trivy-scan.sh index 3dd58ebdce..78862a5574 100755 --- a/tasks/scripts/trivy-scan.sh +++ b/tasks/scripts/trivy-scan.sh @@ -33,7 +33,7 @@ HELM_PROFILES=( cert-manager credential-driver-kubernetes-secrets credential-driver-vault gateway gateway-tls high-availability openshift-route-cert-manager spire tls-disabled workspace-managed workspace-operator - corporate-proxy-e2e keycloak sidecar sidecar-kata skaffold + corporate-proxy-e2e keycloak sandbox-runtime skaffold ) # Reject ignore entries broader than one concrete basename. diff --git a/tasks/test.toml b/tasks/test.toml index 8c41e70025..c5c65b2b07 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -190,9 +190,9 @@ run = [ "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" AGENT_SANDBOX_VERSION=v0.4.6 e2e/rust/e2e-kubernetes.sh", ] -["e2e:kubernetes:sidecar"] -description = "Run Kubernetes e2e with the supervisor sidecar topology overlay" -env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-sidecar.yaml" } +["e2e:kubernetes:isolation"] +description = "Run Kubernetes e2e with the workload network fence and separate supervisor" +env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-sandbox-runtime.yaml" } depends = ["e2e:conformance:build"] run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" @@ -210,13 +210,13 @@ run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debu ["e2e:kubernetes:workspace-managed"] description = "Run Kubernetes e2e with managed workspace mode (auto-created per-workspace namespaces)" -env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET = "e2e-regcred", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" } +env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-sandbox-runtime.yaml:deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET = "e2e-regcred", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" } depends = ["e2e:conformance:build"] run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:workspace-operator"] description = "Run Kubernetes e2e with operator workspace mode (pre-provisioned per-workspace namespaces)" -env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-operator.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_operator", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-operator" } +env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-sandbox-runtime.yaml:deploy/helm/openshell/ci/values-workspace-operator.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_operator", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-operator" } depends = ["e2e:conformance:build"] run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" @@ -248,7 +248,7 @@ depends = ["e2e:conformance:build"] run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-vm.sh" ["e2e:kubernetes:external-driver"] -description = "Run Kubernetes conformance with a driver-free gateway and external Kubernetes driver sidecar" +description = "Run Kubernetes conformance with a driver-free gateway and external Kubernetes driver" env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_KUBE_BUILD_IMAGES = "1", OPENSHELL_E2E_KUBERNETES_FEATURES = "" } depends = ["e2e:conformance:build"] run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"