From f48c82eeda6115b4771959e622b5ab29c37f7214 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Mon, 21 Sep 2026 19:46:44 -0700 Subject: [PATCH 1/8] fix(drivers): require admission labels for external resources Signed-off-by: Drew Newberry --- .agents/skills/helm-dev-environment/SKILL.md | 7 + Cargo.lock | 1 + architecture/compute-runtimes.md | 29 +- crates/openshell-cli/src/main.rs | 8 +- crates/openshell-core/src/lib.rs | 1 + .../openshell-core/src/resource_admission.rs | 383 +++++++++++++ crates/openshell-driver-docker/README.md | 9 +- crates/openshell-driver-docker/src/lib.rs | 209 +++++++- crates/openshell-driver-docker/src/main.rs | 10 +- crates/openshell-driver-docker/src/tests.rs | 118 ++++ crates/openshell-driver-kubernetes/README.md | 7 + .../openshell-driver-kubernetes/src/config.rs | 7 + .../openshell-driver-kubernetes/src/driver.rs | 502 +++++++++++++++++- crates/openshell-driver-kubernetes/src/lib.rs | 1 + .../openshell-driver-kubernetes/src/main.rs | 9 + .../src/resource_admission.rs | 349 ++++++++++++ crates/openshell-driver-mxc/README.md | 6 + crates/openshell-driver-mxc/src/driver.rs | 64 ++- crates/openshell-driver-podman/README.md | 11 +- crates/openshell-driver-podman/src/client.rs | 66 ++- crates/openshell-driver-podman/src/config.rs | 10 + .../openshell-driver-podman/src/container.rs | 46 +- crates/openshell-driver-podman/src/driver.rs | 327 +++++++++++- crates/openshell-driver-podman/src/main.rs | 9 + crates/openshell-driver-vm/README.md | 7 + crates/openshell-driver-vm/src/driver.rs | 47 +- crates/openshell-driver-vm/src/main.rs | 9 + crates/openshell-gateway/Cargo.toml | 1 + crates/openshell-gateway/src/vm.rs | 13 + .../src/compute/driver_config.rs | 55 ++ crates/openshell-server/src/compute/mod.rs | 201 ++++++- crates/openshell-server/src/grpc/sandbox.rs | 4 + crates/openshell-server/src/lib.rs | 5 + crates/openshell-server/src/test_support.rs | 7 + .../openshell-workspace/templates/role.yaml | 4 + deploy/helm/openshell/README.md | 3 + .../helm/openshell/templates/clusterrole.yaml | 11 + .../openshell/templates/gateway-config.yaml | 10 + deploy/helm/openshell/templates/role.yaml | 4 + .../openshell/tests/gateway_config_test.yaml | 39 ++ deploy/helm/openshell/values.yaml | 7 + docs/kubernetes/sandbox-runtime.mdx | 13 + docs/reference/gateway-config.mdx | 117 ++++ docs/reference/sandbox-compute-drivers.mdx | 39 +- proto/compute_driver.proto | 4 + skills/debug-openshell-cluster/SKILL.md | 16 + skills/openshell-cli/SKILL.md | 11 +- 47 files changed, 2733 insertions(+), 83 deletions(-) create mode 100644 crates/openshell-core/src/resource_admission.rs create mode 100644 crates/openshell-driver-kubernetes/src/resource_admission.rs diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 077f70ccd8..2df971436f 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -69,6 +69,13 @@ mise run helm:skaffold:dev mise run helm:skaffold:run ``` +Resource admission defaults to enabled and caller driver config to disabled. +Driver-config scenarios need an explicit `allowDriverConfig` opt-in; external +attachments also need administrator-controlled approval labels in the target +namespace. GPU attachments are exempt from labels. Managed workspace image-pull +Secrets are copied from an approved source in the gateway namespace; do not grant +approval to the gateway database PVC or disable admission to make tests pass. + The Skaffold flow builds distinct `gateway`, `sandbox`, and `supervisor` images and deploys the OpenShell Helm chart. The Kubernetes driver creates a capability-free workload Pod and a directly managed capability-free supervisor diff --git a/Cargo.lock b/Cargo.lock index fe72444b6f..9c2fed8a92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4382,6 +4382,7 @@ dependencies = [ "openshell-server", "rustix 1.1.4", "serde", + "serde_json", "tempfile", "tokio", "toml", diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 1ed9f9d900..44415009c8 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -16,6 +16,22 @@ the common protocol owns process, identity, TCP, DNS, and forwarding semantics. ## Driver Contract +External resource admission is an operator-owned boundary shared by drivers. +The gateway gates caller driver JSON independently from attachment approval. +Drivers resolve the complete effective attachment inventory against authoritative +resource labels before launch and on reuse. Missing labels or an unsupported +resolver deny access; GPU attachments are an explicit temporary exception. +Fresh sandbox-private resources instead require verified provisioning ownership. +Workload metadata must not grant approval or override admission evidence. + +The shared evaluator lives in `openshell-core`; native resolution remains in +each driver. External drivers acknowledge the effective versioned policy through +capabilities, and policy mismatch prevents activation or new launch operations. +Trusted deployment configuration can explicitly disable label admission, but +that opt-out does not waive other ownership and isolation checks. This boundary +assumes operators control approval metadata and runtime resource replacement; +it does not provide atomic mount authorization or instantaneous revocation. + Each runtime receives a sandbox spec and canonical policy from the gateway and is responsible for: @@ -449,11 +465,14 @@ management. RBAC uses a namespace-scoped Role. Each new namespace receives a ServiceAccount and the configured gateway-only SSH ingress NetworkPolicy. Configured image-pull Secrets are copied from the driver's source namespace on every sandbox create so registry credential -rotations propagate. The namespace also copies OpenShift SCC UID-range and -supplemental-group annotations from the gateway namespace when present. The -driver deletes the namespace during workspace deletion. The workspace remains -durably `Terminating` until the Kubernetes API accepts namespace cleanup, so a -transient failure can be retried. Namespace deletion uses the fetched UID as a +rotations propagate. Resource admission first validates the source as shared +operator infrastructure, and copies carry gateway and workspace ownership +labels; an unrelated existing target is never adopted. The namespace also copies +OpenShift SCC UID-range and supplemental-group annotations from the gateway +namespace when present. The driver deletes the namespace during workspace +deletion. The workspace remains durably `Terminating` until the Kubernetes API +accepts namespace cleanup, so a transient failure can be retried. Namespace +deletion uses the fetched UID as a precondition to avoid deleting a replacement namespace. Requires a non-empty `gateway_id` (validated as a DNS-1123 label at startup) so the namespace prefix fits within the K8s 63-character diff --git a/crates/openshell-cli/src/main.rs b/crates/openshell-cli/src/main.rs index 778cc7dfd1..06ca33c9dc 100644 --- a/crates/openshell-cli/src/main.rs +++ b/crates/openshell-cli/src/main.rs @@ -1465,8 +1465,9 @@ enum SandboxCommands { #[arg(long)] memory: Option, - /// Experimental driver-keyed JSON object for driver-specific sandbox settings. - /// Validation behavior is not yet finalized. + /// Driver-keyed JSON object for driver-specific sandbox settings. + /// Disabled unless the gateway administrator enables `allow_driver_config`. + /// External resource attachments still require approval labels. /// /// For Kubernetes, pass a value such as /// `{"kubernetes":{"pod":{"node_selector":{"pool":"gpu"}}}}`. @@ -1885,7 +1886,8 @@ enum SandboxTemplateCommands { #[arg(long, num_args = 0..=1, value_name = "COUNT", default_missing_value = "", value_parser = parse_gpu_request)] gpu: Option, - /// Experimental driver-keyed JSON object for driver-specific sandbox settings. + /// Driver-keyed JSON object for driver-specific sandbox settings. + /// Requires administrator opt-in; resource admission still applies. #[arg(long, value_name = "JSON")] driver_config_json: Option, diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index b310aba8ac..472175f511 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -45,6 +45,7 @@ pub mod proposals; pub mod proto; pub mod proto_struct; pub mod provider_credentials; +pub mod resource_admission; pub mod rpc_error; pub mod sandbox_env; pub mod sandbox_generation; diff --git a/crates/openshell-core/src/resource_admission.rs b/crates/openshell-core/src/resource_admission.rs new file mode 100644 index 0000000000..05c4b81318 --- /dev/null +++ b/crates/openshell-core/src/resource_admission.rs @@ -0,0 +1,383 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Operator-owned external resource admission. GPU attachments are temporarily +//! exempt; this policy must not be interpreted as permission to attach host paths. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +const WORKSPACE_PLACEHOLDER: &str = "${workspace}"; + +/// Label policy shared by every compute driver. A supplied map replaces defaults. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct ResourceAdmissionConfig { + pub enabled: bool, + pub required_labels: BTreeMap, +} + +impl Default for ResourceAdmissionConfig { + fn default() -> Self { + Self { + enabled: true, + required_labels: BTreeMap::from([ + ("openshell.ai/sandbox-attachable".into(), "true".into()), + ( + "openshell.ai/workspace".into(), + WORKSPACE_PLACEHOLDER.into(), + ), + ]), + } + } +} + +/// Effective policy acknowledged by a driver, including the independent JSON gate. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct DriverAdmissionConfig { + pub allow_driver_config: bool, + pub resource_admission: ResourceAdmissionConfig, +} + +impl DriverAdmissionConfig { + pub fn validate(&self) -> Result<(), String> { + self.resource_admission.validate() + } + + /// Versioned acknowledgement. Compare parsed policies, never an untrusted flag. + #[must_use] + pub fn acknowledgement(&self) -> String { + // These types contain only strings, booleans and string-keyed maps. + format!( + "v1:{}", + serde_json::to_string(self).expect("serializable admission policy") + ) + } + + pub fn verify_acknowledgement(&self, value: &str) -> Result<(), String> { + self.validate()?; + if !self.resource_admission.enabled && value.is_empty() { + // Explicit legacy-driver opt-out. The gateway still gates caller JSON. + return Ok(()); + } + let policy: Self = value + .strip_prefix("v1:") + .ok_or("compute driver does not acknowledge resource admission v1") + .and_then(|json| { + serde_json::from_str(json).map_err(|_| "invalid driver admission acknowledgement") + })?; + if policy != *self { + return Err( + "compute driver admission policy differs from gateway configuration".into(), + ); + } + Ok(()) + } +} + +impl std::str::FromStr for DriverAdmissionConfig { + type Err = String; + fn from_str(value: &str) -> Result { + let policy: Self = serde_json::from_str(value).map_err(|error| error.to_string())?; + policy.validate()?; + Ok(policy) + } +} + +/// Reserved driver-owned runtime metadata; caller labels must never override it. +pub const CONFIG_USED_LABEL: &str = "openshell.ai/caller-driver-config-used"; +pub const IDENTITIES_LABEL: &str = "openshell.ai/resource-admission-identities"; + +pub fn check_config_provenance(allowed: bool, recorded: Option<&str>) -> Result<(), tonic::Status> { + match recorded { + Some("false") => Ok(()), + Some("true") if allowed => Ok(()), + _ => Err(tonic::Status::failed_precondition( + "sandbox uses forbidden driver config or lacks admission provenance; recreate it", + )), + } +} + +/// Check caller config before resource lookups, including direct driver RPCs. +pub fn check_driver_config( + allowed: bool, + config: Option<&prost_types::Struct>, +) -> Result<(), tonic::Status> { + if !allowed && config.is_some_and(|config| !config.fields.is_empty()) { + return Err(tonic::Status::failed_precondition( + "caller driver config is disabled; a gateway administrator must enable allow_driver_config", + )); + } + Ok(()) +} + +pub fn check_sandbox_driver_config( + allowed: bool, + sandbox: &crate::proto::compute::v1::DriverSandbox, +) -> Result<(), tonic::Status> { + check_driver_config( + allowed, + sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()) + .and_then(|template| template.driver_config.as_ref()), + ) +} + +impl ResourceAdmissionConfig { + pub fn validate(&self) -> Result<(), String> { + if self.enabled && self.required_labels.is_empty() { + return Err( + "resource_admission.required_labels must not be empty while enabled".into(), + ); + } + for (key, value) in &self.required_labels { + if !valid_label_key(key) { + return Err(format!("invalid resource admission label key: {key}")); + } + if value != WORKSPACE_PLACEHOLDER && !valid_label_value(value) { + return Err(format!( + "invalid resource admission label value for {key}; only whole-value ${{workspace}} substitution is supported" + )); + } + } + if self.enabled + && self + .required_labels + .values() + .all(|value| value == WORKSPACE_PLACEHOLDER) + { + return Err( + "resource_admission.required_labels must include a shared approval label".into(), + ); + } + Ok(()) + } + + /// Evaluate labels read from the resource authority, never caller metadata. + pub fn admit<'a>( + &self, + workspace: &str, + labels: impl IntoIterator, + ) -> Result<(), tonic::Status> { + self.admit_labels(Some(workspace), labels) + } + + /// Evaluate a shared operator resource. Fixed approval labels still apply, + /// but workspace placeholders do not: cluster-scoped and intentionally + /// shared resources cannot carry one tenant's identity. + pub fn admit_shared<'a>( + &self, + labels: impl IntoIterator, + ) -> Result<(), tonic::Status> { + self.admit_labels(None, labels) + } + + fn admit_labels<'a>( + &self, + workspace: Option<&str>, + labels: impl IntoIterator, + ) -> Result<(), tonic::Status> { + self.validate() + .map_err(tonic::Status::failed_precondition)?; + if !self.enabled { + return Ok(()); + } + let labels: BTreeMap<_, _> = labels.into_iter().collect(); + for (key, expected) in &self.required_labels { + let expected = if expected == WORKSPACE_PLACEHOLDER { + let Some(workspace) = workspace else { + continue; + }; + if workspace.is_empty() || !valid_label_value(workspace) { + return Err(tonic::Status::failed_precondition( + "resource not admitted: invalid workspace identity", + )); + } + workspace + } else { + expected + }; + if labels.get(key).map(|value| value.as_str()) != Some(expected) { + return Err(tonic::Status::failed_precondition( + "external resource not admitted by required labels", + )); + } + } + Ok(()) + } + + pub fn reject_unlabelable(&self, kind: &str) -> Result<(), tonic::Status> { + if self.enabled { + Err(tonic::Status::failed_precondition(format!( + "{kind} cannot be attached while resource admission is enabled: no trusted label resolver" + ))) + } else { + Ok(()) + } + } +} + +fn valid_label_value(value: &str) -> bool { + value.is_empty() + || (value.len() <= 63 + && value + .as_bytes() + .first() + .is_some_and(u8::is_ascii_alphanumeric) + && value + .as_bytes() + .last() + .is_some_and(u8::is_ascii_alphanumeric) + && value + .bytes() + .all(|c| c.is_ascii_alphanumeric() || b"-_.".contains(&c))) +} + +fn valid_label_key(key: &str) -> bool { + let (prefix, name) = key + .split_once('/') + .map_or((None, key), |(prefix, name)| (Some(prefix), name)); + !name.is_empty() + && valid_label_value(name) + && prefix.is_none_or(|prefix| { + !prefix.is_empty() + && prefix.len() <= 253 + && prefix.split('.').all(|part| { + !part.is_empty() + && part.len() <= 63 + && part + .bytes() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == b'-') + && part + .as_bytes() + .first() + .is_some_and(u8::is_ascii_alphanumeric) + && part + .as_bytes() + .last() + .is_some_and(u8::is_ascii_alphanumeric) + }) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn labels(workspace: &str) -> BTreeMap { + BTreeMap::from([ + ("openshell.ai/sandbox-attachable".into(), "true".into()), + ("openshell.ai/workspace".into(), workspace.into()), + ]) + } + + #[test] + fn rejects_unapproved_and_cross_workspace_resources() { + let policy = ResourceAdmissionConfig::default(); + assert!(policy.admit("a", &labels("a")).is_ok()); + assert!(policy.admit("b", &labels("a")).is_err()); + assert!(policy.admit_shared(&labels("a")).is_ok()); + assert!(policy.admit("a", &BTreeMap::new()).is_err()); + assert!(policy.admit_shared(&BTreeMap::new()).is_err()); + assert!(policy.admit("", &labels("")).is_err()); + } + + #[test] + fn replacement_map_and_explicit_empty_policy() { + let policy: ResourceAdmissionConfig = + serde_json::from_str(r#"{"required_labels":{"example.com/approved":"yes"}}"#).unwrap(); + assert_eq!(policy.required_labels.len(), 1); + assert!( + policy + .admit( + "a", + &BTreeMap::from([("example.com/approved".into(), "yes".into())]) + ) + .is_ok() + ); + let empty: ResourceAdmissionConfig = + serde_json::from_str(r#"{"required_labels":{}}"#).unwrap(); + assert!(empty.validate().is_err()); + let workspace_only: ResourceAdmissionConfig = serde_json::from_str( + r#"{"required_labels":{"openshell.ai/workspace":"${workspace}"}}"#, + ) + .unwrap(); + assert!(workspace_only.validate().is_err()); + assert!( + ResourceAdmissionConfig { + enabled: false, + ..empty + } + .validate() + .is_ok() + ); + } + + #[test] + fn rejects_invalid_labels_and_interpolation() { + for value in [ + "${workspace.id}", + "prefix-${workspace}", + "$HOME", + "*", + "x/y", + ] { + let policy = ResourceAdmissionConfig { + required_labels: BTreeMap::from([("approved".into(), value.into())]), + ..Default::default() + }; + assert!(policy.validate().is_err(), "{value}"); + } + for key in ["", "/a", "bad_domain/a", "a/b/c", "-a", "a/"] { + assert!(!valid_label_key(key), "{key}"); + } + } + + #[test] + fn json_gate_is_independent_of_label_opt_out() { + let config = prost_types::Struct { + fields: BTreeMap::from([("gpu_device_ids".into(), prost_types::Value::default())]), + }; + for enabled in [false, true] { + let policy = DriverAdmissionConfig { + resource_admission: ResourceAdmissionConfig { + enabled, + ..Default::default() + }, + ..Default::default() + }; + assert!(check_driver_config(policy.allow_driver_config, Some(&config)).is_err()); + assert!(check_driver_config(true, Some(&config)).is_ok()); + assert!(check_driver_config(false, Some(&prost_types::Struct::default())).is_ok()); + assert!(check_driver_config(false, None).is_ok()); + } + } + + #[test] + fn driver_acknowledgement_must_match_effective_policy() { + let policy = DriverAdmissionConfig::default(); + assert!( + policy + .verify_acknowledgement(&policy.acknowledgement()) + .is_ok() + ); + assert!(policy.verify_acknowledgement("").is_err()); + let unsafe_policy = DriverAdmissionConfig { + resource_admission: ResourceAdmissionConfig { + enabled: false, + ..Default::default() + }, + ..Default::default() + }; + assert!( + policy + .verify_acknowledgement(&unsafe_policy.acknowledgement()) + .is_err() + ); + assert!(unsafe_policy.verify_acknowledgement("").is_ok()); + } +} diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index aa45b14745..096181a69f 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -15,6 +15,12 @@ metadata, and flushes spans during graceful shutdown. ## Runtime Model +Caller driver config is disabled by default. Existing volumes require +administrator-controlled approval labels; raw bind mounts have no supported +label resolver and are denied under enforcement. GPU devices are temporarily +exempt. Inspect labels again before launch, restart, and during reconciliation. +See [resource admission configuration](../../docs/reference/gateway-config.mdx#external-resource-admission). + The driver creates two containers for each sandbox: - `openshell-sandbox` is PID 1 in the workload container. It owns the workload @@ -98,7 +104,8 @@ The gateway forwards the `docker` block from `--driver-config-json`. Supported mount types are: - `bind`: an absolute daemon-host path, allowed only when - `[openshell.drivers.docker].enable_bind_mounts = true`. + `[openshell.drivers.docker].enable_bind_mounts = true` and label admission + is explicitly disabled. - `volume`: an existing named volume. The driver never creates or removes a user-supplied volume. Bind-backed local volumes require `enable_bind_mounts = true`. diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 9bef89f353..fd41581c67 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -154,6 +154,10 @@ fn provisioning_span( #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] #[serde(default, deny_unknown_fields)] pub struct DockerComputeConfig { + /// Permit caller-supplied driver JSON. Does not waive resource admission. + pub allow_driver_config: bool, + /// Operator-owned external attachment approval policy. + pub resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig, /// Docker API Unix socket. When unset, use the socket selected by gateway /// auto-detection, falling back to `/var/run/docker.sock` for an explicitly /// configured Docker driver. @@ -225,6 +229,7 @@ pub struct DockerComputeConfig { impl DockerComputeConfig { /// Validate startup configuration without connecting to Docker. pub fn validate_configuration(&self, gateway_bind_address: SocketAddr) -> CoreResult<()> { + self.resource_admission.validate().map_err(Error::config)?; if let Some(socket_path) = self.socket_path.as_deref() && socket_path.to_str().is_none() { @@ -254,6 +259,9 @@ impl Default for DockerComputeConfig { fn default() -> Self { Self { socket_path: None, + allow_driver_config: false, + resource_admission: + openshell_core::resource_admission::ResourceAdmissionConfig::default(), default_image: openshell_core::image::default_sandbox_image(), image_pull_policy: ImagePullPolicy::default(), sandbox_label: "default".to_string(), @@ -283,6 +291,8 @@ pub(crate) struct DockerGuestTlsPaths { #[derive(Debug, Clone)] struct DockerDriverRuntimeConfig { + allow_driver_config: bool, + resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig, default_image: String, image_pull_policy: ImagePullPolicy, sandbox_namespace: String, @@ -827,6 +837,10 @@ impl DockerComputeDriver { gateway_log_level: &str, docker_config: &DockerComputeConfig, ) -> CoreResult { + docker_config + .resource_admission + .validate() + .map_err(Error::config)?; let socket_path = docker_config .socket_path .clone() @@ -930,6 +944,8 @@ impl DockerComputeDriver { gpu, sandbox_pids_limit: docker_config.sandbox_pids_limit, enable_bind_mounts: docker_config.enable_bind_mounts, + allow_driver_config: docker_config.allow_driver_config, + resource_admission: docker_config.resource_admission.clone(), upstream_proxy: docker_config.upstream_proxy.clone(), provider_spiffe_workload_api_socket: docker_config .provider_spiffe_workload_api_socket @@ -966,6 +982,11 @@ impl DockerComputeDriver { fn capabilities(&self) -> GetCapabilitiesResponse { GetCapabilitiesResponse { + resource_admission_policy: openshell_core::resource_admission::DriverAdmissionConfig { + allow_driver_config: self.config.allow_driver_config, + resource_admission: self.config.resource_admission.clone(), + } + .acknowledgement(), driver_name: "docker".to_string(), driver_version: openshell_core::VERSION.to_string(), default_image: self.config.default_image.clone(), @@ -1008,6 +1029,10 @@ impl DockerComputeDriver { sandbox: &'a DriverSandbox, config: &DockerDriverRuntimeConfig, ) -> Result, Status> { + openshell_core::resource_admission::check_sandbox_driver_config( + config.allow_driver_config, + sandbox, + )?; let spec = sandbox .spec .as_ref() @@ -1022,6 +1047,16 @@ impl DockerComputeDriver { let driver_config = DockerSandboxDriverConfig::from_template(template).map_err(Status::invalid_argument)?; validate_docker_driver_mounts(&driver_config.mounts, config.enable_bind_mounts)?; + for mount in &driver_config.mounts { + if matches!( + mount, + DockerDriverMountConfig::Bind { .. } | DockerDriverMountConfig::Image { .. } + ) { + config + .resource_admission + .reject_unlabelable("host bind or image mount")?; + } + } let gpu_requirements = driver_gpu_requirements(spec.resource_requirements.as_ref()); Self::validate_gpu_request(gpu_requirements, config.gpu.cdi_supported, &driver_config)?; Ok(ValidatedDockerSandbox { @@ -1096,11 +1131,17 @@ impl DockerComputeDriver { async fn validate_user_volume_mounts_available( &self, driver_config: &DockerSandboxDriverConfig, - ) -> Result<(), Status> { + workspace: &str, + ) -> Result, Status> { + let mut identities = std::collections::BTreeMap::new(); for mount in &driver_config.mounts { if let DockerDriverMountConfig::Volume { source, .. } = mount { match self.docker.inspect_volume(source).await { Ok(volume) => { + identities.insert(source.clone(), docker_volume_identity(&volume)); + self.config + .resource_admission + .admit(workspace, &volume.labels)?; if !self.config.enable_bind_mounts && docker_volume_is_bind_backed(&volume) { return Err(Status::failed_precondition(format!( @@ -1119,7 +1160,7 @@ impl DockerComputeDriver { } } } - Ok(()) + Ok(identities) } async fn refresh_gpu_inventory(&self) -> Result<(), Status> { @@ -1135,6 +1176,108 @@ impl DockerComputeDriver { Ok(()) } + async fn admit_container_resources( + &self, + container: &bollard::models::ContainerInspectResponse, + ) -> Result<(), Status> { + if self.config.allow_driver_config && !self.config.resource_admission.enabled { + return Ok(()); + } + let labels = container + .config + .as_ref() + .and_then(|config| config.labels.as_ref()) + .ok_or_else(|| Status::failed_precondition("sandbox lacks admission metadata"))?; + openshell_core::resource_admission::check_config_provenance( + self.config.allow_driver_config, + labels + .get(openshell_core::resource_admission::CONFIG_USED_LABEL) + .map(String::as_str), + )?; + if !self.config.resource_admission.enabled { + return Ok(()); + } + let workspace = labels + .get(LABEL_SANDBOX_WORKSPACE) + .ok_or_else(|| Status::failed_precondition("sandbox lacks workspace"))?; + let sandbox_id = labels + .get(LABEL_SANDBOX_ID) + .ok_or_else(|| Status::failed_precondition("sandbox lacks identity"))?; + let expected: std::collections::BTreeMap = labels + .get(openshell_core::resource_admission::IDENTITIES_LABEL) + .and_then(|value| serde_json::from_str(value).ok()) + .ok_or_else(|| Status::failed_precondition("sandbox lacks resource identity record"))?; + let mut actual = std::collections::BTreeMap::new(); + let anonymous_targets: Vec = labels + .get("openshell.ai/private-image-volume-targets") + .and_then(|value| serde_json::from_str(value).ok()) + .unwrap_or_default(); + for mount in container.mounts.as_deref().unwrap_or_default() { + match mount.typ { + Some(bollard::models::MountPointTypeEnum::VOLUME) => { + let name = mount + .name + .as_deref() + .ok_or_else(|| Status::failed_precondition("volume lacks identity"))?; + let volume = self.docker.inspect_volume(name).await.map_err(|error| { + if is_not_found_error(&error) { + Status::failed_precondition("attached volume no longer exists") + } else { + Status::unavailable("volume admission lookup failed") + } + })?; + if name == docker_channel_volume_name_by_id(sandbox_id, &self.config) { + if volume.driver != "local" + || !volume.options.is_empty() + || volume.labels.get(LABEL_SANDBOX_ID) != Some(sandbox_id) + || volume.labels.get(LABEL_MANAGED_BY).map(String::as_str) + != Some(LABEL_MANAGED_BY_VALUE) + { + return Err(Status::failed_precondition( + "sandbox-private volume ownership changed", + )); + } + } else if !expected.contains_key(name) + && mount + .destination + .as_ref() + .is_some_and(|destination| anonymous_targets.contains(destination)) + { + // The immutable create spec requested a fresh anonymous + // image volume at this target, never an existing name. + if volume.driver != "local" || !volume.options.is_empty() { + return Err(Status::failed_precondition( + "image-private volume backing changed", + )); + } + } else { + actual.insert(name.to_string(), docker_volume_identity(&volume)); + self.config + .resource_admission + .admit(workspace, &volume.labels)?; + if !self.config.enable_bind_mounts && docker_volume_is_bind_backed(&volume) + { + return Err(Status::failed_precondition( + "bind-backed volume is disabled", + )); + } + } + } + Some(bollard::models::MountPointTypeEnum::TMPFS) => {} + _ => self + .config + .resource_admission + .reject_unlabelable("effective container mount")?, + } + } + if actual != expected { + return Err(Status::failed_precondition( + "external volume identity or attachment inventory changed", + )); + } + Ok(()) + } + async fn resolve_gpu_cdi_devices( &self, gpu_requirements: Option<&GpuResourceRequirements>, @@ -1273,7 +1416,13 @@ impl DockerComputeDriver { } } Ok(inspected) if summary.state == Some(ContainerSummaryStateEnum::RUNNING) => { - if let Err(status) = validate_docker_outer_fence(&inspected) { + let admission = match validate_docker_outer_fence(&inspected) { + Ok(()) => self.admit_container_resources(&inspected).await, + Err(error) => Err(error), + }; + if let Err(status) = admission + && status.code() == tonic::Code::FailedPrecondition + { let context = self .control_failure_context(sandbox.clone(), container_id.to_string()); handle_docker_runtime_failure( @@ -1313,7 +1462,7 @@ impl DockerComputeDriver { async fn create_sandbox_inner(&self, sandbox: &DriverSandbox) -> Result<(), Status> { let validated = Self::validated_sandbox(sandbox, &self.config)?; Self::validate_sandbox_auth(sandbox)?; - self.validate_user_volume_mounts_available(&validated.driver_config) + self.validate_user_volume_mounts_available(&validated.driver_config, &sandbox.workspace) .await?; let _ = self .resolve_gpu_cdi_devices( @@ -1472,7 +1621,7 @@ impl DockerComputeDriver { )); } }; - let create_body = match build_container_create_body_for_image( + let mut create_body = match build_container_create_body_for_image( sandbox, &self.config, &validated.driver_config, @@ -1491,6 +1640,30 @@ impl DockerComputeDriver { )); } }; + let identities = match self + .validate_user_volume_mounts_available(&validated.driver_config, &sandbox.workspace) + .await + { + Ok(identities) => identities, + Err(status) => { + let _ = remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::from_status( + "ResourceAdmissionDenied", + status, + )); + } + }; + create_body + .labels + .get_or_insert_with(Default::default) + .insert( + openshell_core::resource_admission::IDENTITIES_LABEL.into(), + serde_json::to_string(&identities).map_err(|error| { + DockerProvisioningFailure::new("ResourceAdmissionDenied", error.to_string()) + })?, + ); let create_result = async { openshell_otel::record_error_result( self.docker @@ -1544,7 +1717,10 @@ impl DockerComputeDriver { )); } }; - let outer_fence_error = validate_docker_outer_fence(&inspected).err(); + let outer_fence_error = match validate_docker_outer_fence(&inspected) { + Ok(()) => self.admit_container_resources(&inspected).await.err(), + Err(error) => Some(error), + }; drop(inspected); if let Some(status) = outer_fence_error { let _ = self @@ -2224,6 +2400,7 @@ impl DockerComputeDriver { .await .map_err(|error| internal_status("inspect Docker sandbox outer fence", error))?; validate_docker_outer_fence(&inspected)?; + self.admit_container_resources(&inspected).await?; let state = container.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); let previous_finished_at = if state == ContainerSummaryStateEnum::EXITED { inspected @@ -3037,7 +3214,7 @@ impl ComputeDriver for DockerComputeDriver { .sandbox .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; let validated = Self::validated_sandbox(&sandbox, &self.config)?; - self.validate_user_volume_mounts_available(&validated.driver_config) + self.validate_user_volume_mounts_available(&validated.driver_config, &sandbox.workspace) .await?; let _ = self .resolve_gpu_cdi_devices( @@ -3779,6 +3956,11 @@ fn docker_tmpfs_option(option: &str) -> Result, Status> { } } +fn docker_volume_identity(volume: &bollard::models::Volume) -> serde_json::Value { + serde_json::json!({"name": volume.name, "driver": volume.driver, "options": volume.options, + "created_at": volume.created_at, "mountpoint": volume.mountpoint}) +} + fn docker_volume_is_bind_backed(volume: &bollard::models::Volume) -> bool { volume.driver == "local" && volume.options.get("o").is_some_and(|options| { @@ -5503,6 +5685,19 @@ fn build_container_create_body_for_image( }] }); let mut labels = template.labels.clone(); + labels.insert( + "openshell.ai/private-image-volume-targets".into(), + serde_json::to_string(&image.volumes) + .map_err(|error| Status::internal(error.to_string()))?, + ); + labels.insert( + openshell_core::resource_admission::CONFIG_USED_LABEL.into(), + template + .driver_config + .as_ref() + .is_some_and(|config| !config.fields.is_empty()) + .to_string(), + ); labels.insert( LABEL_MANAGED_BY.to_string(), LABEL_MANAGED_BY_VALUE.to_string(), diff --git a/crates/openshell-driver-docker/src/main.rs b/crates/openshell-driver-docker/src/main.rs index 8630d37c11..746c62981c 100644 --- a/crates/openshell-driver-docker/src/main.rs +++ b/crates/openshell-driver-docker/src/main.rs @@ -14,6 +14,9 @@ use tracing::info; #[derive(Debug, Parser)] #[command(name = "openshell-driver-docker", version = VERSION)] struct Args { + /// Override the operator admission policy in the driver TOML file. + #[arg(long, env = "OPENSHELL_DRIVER_ADMISSION_CONFIG_JSON")] + admission_config_json: Option, /// Public compute-driver Unix socket used by the gateway. #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] bind_socket: PathBuf, @@ -54,7 +57,12 @@ async fn main() -> Result<()> { ); let config_source = std::fs::read_to_string(&args.config).into_diagnostic()?; - let docker_config: DockerComputeConfig = toml::from_str(&config_source).into_diagnostic()?; + let mut docker_config: DockerComputeConfig = + toml::from_str(&config_source).into_diagnostic()?; + if let Some(policy) = args.admission_config_json { + docker_config.allow_driver_config = policy.allow_driver_config; + docker_config.resource_admission = policy.resource_admission; + } let driver = DockerComputeDriver::new(args.gateway_bind, &args.log_level, &docker_config) .await .into_diagnostic()?; diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 0908fc2739..0aa8f5c5c7 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -79,6 +79,58 @@ fn test_sandbox() -> DriverSandbox { } } +#[test] +fn admission_defaults_reject_even_gpu_driver_json_but_not_public_gpu_requests() { + let mut config = runtime_config(); + config.allow_driver_config = false; + config.resource_admission = + openshell_core::resource_admission::ResourceAdmissionConfig::default(); + config.gpu.cdi_supported = true; + let mut sandbox = test_sandbox(); + let spec = sandbox.spec.as_mut().unwrap(); + spec.resource_requirements = Some(gpu_resources(Some(1))); + assert!(DockerComputeDriver::validate_sandbox(&sandbox, &config).is_ok()); + sandbox + .spec + .as_mut() + .unwrap() + .template + .as_mut() + .unwrap() + .driver_config = Some(cdi_devices_config(&["nvidia.com/gpu=0"])); + let error = DockerComputeDriver::validate_sandbox(&sandbox, &config).unwrap_err(); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + config.allow_driver_config = true; + assert!(DockerComputeDriver::validate_sandbox(&sandbox, &config).is_ok()); +} + +#[test] +fn admission_blocks_unlabelable_bind_mount_even_when_bind_mounts_enabled() { + let mut config = runtime_config(); + config.enable_bind_mounts = true; + config.resource_admission = + openshell_core::resource_admission::ResourceAdmissionConfig::default(); + let mut sandbox = test_sandbox(); + sandbox + .spec + .as_mut() + .unwrap() + .template + .as_mut() + .unwrap() + .driver_config = Some( + openshell_core::proto_struct::json_object_to_struct( + serde_json::json!({"mounts":[{"type":"bind","source":"/srv/data","target":"/data"}]}) + .as_object() + .unwrap() + .clone(), + ) + .unwrap(), + ); + let error = DockerComputeDriver::validate_sandbox(&sandbox, &config).unwrap_err(); + assert!(error.message().contains("no trusted label resolver")); +} + fn cdi_devices_config(device_ids: &[&str]) -> prost_types::Struct { list_string_driver_config("cdi_devices", device_ids) } @@ -118,6 +170,12 @@ fn gpu_resources(count: Option) -> ResourceRequirements { fn runtime_config() -> DockerDriverRuntimeConfig { DockerDriverRuntimeConfig { + // Existing lifecycle fixtures exercise the explicitly opted-out contract. + allow_driver_config: true, + resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig { + enabled: false, + ..Default::default() + }, default_image: "image:latest".to_string(), image_pull_policy: ImagePullPolicy::IfNotPresent, sandbox_namespace: "default".to_string(), @@ -237,6 +295,66 @@ async fn capabilities_reject_missing_gateway_metadata() { ); } +#[tokio::test] +#[ignore = "requires a local Docker daemon; creates and removes only isolated test volumes"] +async fn live_docker_resource_admission_checks_native_volume_labels() { + let temporary = TempDir::new().unwrap(); + let suffix = temporary.path().file_name().unwrap().to_str().unwrap(); + let mut config = runtime_config(); + config.resource_admission = + openshell_core::resource_admission::ResourceAdmissionConfig::default(); + let mut driver = test_driver_with_config(config); + driver.docker = Arc::new(Docker::connect_with_local_defaults().unwrap()); + for (index, (workspace, approved)) in [ + (None, false), + (Some("other"), false), + (Some("team-a"), true), + ] + .into_iter() + .enumerate() + { + let name = format!("openshell-admission-{suffix}-{index}"); + assert!(driver.docker.inspect_volume(&name).await.is_err()); + let labels = workspace.map(|workspace| { + HashMap::from([ + ("openshell.ai/sandbox-attachable".into(), "true".into()), + ("openshell.ai/workspace".into(), workspace.into()), + ]) + }); + driver + .docker + .create_volume(VolumeCreateRequest { + name: Some(name.clone()), + labels, + ..Default::default() + }) + .await + .unwrap(); + let mut results = Vec::new(); + for read_only in [true, false] { + let mounts = serde_json::from_value(serde_json::json!({"mounts":[{ + "type":"volume", "source":name, "target":"/external", "read_only":read_only + }]})) + .unwrap(); + results.push( + driver + .validate_user_volume_mounts_available(&mounts, "team-a") + .await + .is_ok(), + ); + } + driver + .docker + .remove_volume( + &name, + None::, + ) + .await + .unwrap(); + assert_eq!(results, vec![approved; 2]); + } +} + type TestDriverClient = openshell_core::proto::compute::v1::compute_driver_client::ComputeDriverClient< tonic::transport::Channel, diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 07f8a0ffe1..eedece1a1c 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -2,6 +2,13 @@ Kubernetes-backed compute driver for OpenShell cluster deployments. +Caller driver config is disabled by default. External resource references need +administrator-controlled approval labels in every workspace mode, including +before restart and scheduling-gate release. GPU devices are temporarily exempt. +Managed workspace image-pull Secrets are copied only after the configured source +Secret passes shared-resource admission; copies carry gateway ownership metadata. +See [resource admission configuration](../../docs/reference/gateway-config.mdx#external-resource-admission). + The driver uses the Kubernetes API to create, delete, fetch, and watch sandbox custom resources. It runs in-process with the gateway server and supports three workspace namespace modes via `workspace_mode`: diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index f29ba79f5b..ee7f4be8d5 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -153,6 +153,10 @@ where #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct KubernetesComputeConfig { + /// Permit caller-supplied driver JSON. Does not waive resource admission. + pub allow_driver_config: bool, + /// Operator-owned external attachment approval policy. + pub resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig, /// How workspaces map to Kubernetes namespaces. `"shared"` (default) /// renders all sandboxes into `namespace`; `"managed"` creates per-workspace /// namespaces on demand; `"operator"` uses pre-provisioned namespaces. @@ -294,6 +298,9 @@ impl Default for KubernetesComputeConfig { fn default() -> Self { Self { workspace_mode: WorkspaceMode::default(), + allow_driver_config: false, + resource_admission: + openshell_core::resource_admission::ResourceAdmissionConfig::default(), gateway_id: DEFAULT_GATEWAY_ID.to_string(), namespace: DEFAULT_K8S_NAMESPACE.to_string(), operator_namespace_label: None, diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index bd71dd3545..7b64434868 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -228,6 +228,17 @@ impl From for openshell_core::ComputeDriverError { /// This prevents gRPC handlers from blocking indefinitely when the k8s /// API server is unreachable or slow. const KUBE_API_TIMEOUT: Duration = Duration::from_secs(30); +fn admission_error(error: tonic::Status) -> KubernetesDriverError { + match error.code() { + tonic::Code::InvalidArgument => { + KubernetesDriverError::InvalidArgument(error.message().into()) + } + tonic::Code::FailedPrecondition => { + KubernetesDriverError::Precondition(error.message().into()) + } + _ => KubernetesDriverError::Message(error.message().into()), + } +} const SANDBOX_RUNTIME_RECONCILE_INTERVAL: Duration = Duration::from_secs(30); /// Bound how long a crash-interrupted, fail-closed bootstrap may remain stranded. const SANDBOX_RUNTIME_BOOTSTRAP_GRACE: Duration = Duration::from_mins(5); @@ -673,6 +684,10 @@ impl KubernetesComputeDriver { config: KubernetesComputeConfig, shutdown_rx: tokio::sync::watch::Receiver, ) -> Result { + config + .resource_admission + .validate() + .map_err(KubernetesDriverError::Precondition)?; config .validate_workspace_mode() .map_err(KubernetesDriverError::Precondition)?; @@ -747,6 +762,11 @@ impl KubernetesComputeDriver { pub fn capabilities(&self) -> Result { Ok(GetCapabilitiesResponse { + resource_admission_policy: openshell_core::resource_admission::DriverAdmissionConfig { + allow_driver_config: self.config.allow_driver_config, + resource_admission: self.config.resource_admission.clone(), + } + .acknowledgement(), driver_name: "kubernetes".to_string(), driver_version: openshell_core::VERSION.to_string(), default_image: self.config.default_image.clone(), @@ -1215,6 +1235,7 @@ impl KubernetesComputeDriver { async fn ensure_image_pull_secrets( &self, namespace: &str, + workspace: &str, ) -> Result<(), KubernetesDriverError> { let source_api: Api = Api::namespaced(self.client.clone(), &self.config.namespace); let target_api: Api = Api::namespaced(self.client.clone(), namespace); @@ -1239,7 +1260,45 @@ impl KubernetesComputeDriver { } }; - let copy = image_pull_secret_copy(secret_name, namespace, source); + self.config + .resource_admission + .admit_shared( + source + .metadata + .labels + .as_ref() + .into_iter() + .flat_map(|labels| labels.iter()), + ) + .map_err(admission_error)?; + + let existing = tokio::time::timeout(KUBE_API_TIMEOUT, target_api.get_opt(secret_name)) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timeout checking image-pull Secret {secret_name} in {namespace}" + )) + })? + .map_err(KubernetesDriverError::from_kube)?; + if existing.as_ref().is_some_and(|existing| { + !image_pull_secret_owned_by_gateway( + existing.metadata.labels.as_ref(), + &self.config.gateway_id, + workspace, + ) + }) { + return Err(KubernetesDriverError::Precondition(format!( + "image-pull Secret {secret_name} in {namespace} is not owned by this gateway" + ))); + } + + let copy = image_pull_secret_copy( + secret_name, + namespace, + workspace, + &self.config.gateway_id, + source, + ); match tokio::time::timeout( KUBE_API_TIMEOUT, target_api.patch( @@ -1508,6 +1567,14 @@ impl KubernetesComputeDriver { } pub async fn validate_sandbox_create(&self, sandbox: &Sandbox) -> Result<(), tonic::Status> { + openshell_core::resource_admission::check_sandbox_driver_config( + self.config.allow_driver_config, + sandbox, + )?; + self.config + .resource_admission + .validate() + .map_err(tonic::Status::failed_precondition)?; let _ = Self::validate_driver_config_for_sandbox(sandbox) .map_err(tonic::Status::invalid_argument)?; match self.config.workspace_mode { @@ -1536,6 +1603,88 @@ impl KubernetesComputeDriver { Ok(()) } + async fn admit_requested_resources( + &self, + sandbox: &Sandbox, + ) -> Result { + openshell_core::resource_admission::check_sandbox_driver_config( + self.config.allow_driver_config, + sandbox, + )?; + self.validate_workspace_namespace(&sandbox.workspace) + .map_err(|error| tonic::Status::failed_precondition(error.to_string()))?; + let namespace = self + .config + .namespace_for_workspace(&sandbox.workspace, self.operator_allowlist.as_ref()) + .map_err(tonic::Status::failed_precondition)?; + let params = SandboxPodParams { + default_image: &self.config.default_image, + image_pull_secrets: &self.config.image_pull_secrets, + default_runtime_class_name: &self.config.default_runtime_class_name, + service_account_name: &self.config.service_account_name, + ..Default::default() + }; + let rendered = sandbox_to_k8s_spec(sandbox.spec.as_ref(), ¶ms) + .map_err(tonic::Status::invalid_argument)?; + crate::resource_admission::admit( + &self.client, + &self.config.resource_admission, + &sandbox.workspace, + &namespace, + &rendered["spec"]["podTemplate"]["spec"], + params.sandbox_secret_name, + ) + .await + } + + async fn admit_stored_resources(&self, object: &DynamicObject) -> Result<(), tonic::Status> { + if self.config.allow_driver_config && !self.config.resource_admission.enabled { + return Ok(()); + } + let expected = crate::resource_admission::check_record( + object.metadata.annotations.as_ref(), + self.config.allow_driver_config, + )?; + if !self.config.resource_admission.enabled { + return Ok(()); + } + let workspace = object + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_WORKSPACE)) + .ok_or_else(|| { + tonic::Status::failed_precondition("sandbox lacks workspace identity") + })?; + let namespace = object + .metadata + .namespace + .as_deref() + .ok_or_else(|| tonic::Status::failed_precondition("sandbox lacks namespace"))?; + let sandbox_id = + sandbox_id_from_object(object).map_err(tonic::Status::failed_precondition)?; + let generation = sandbox_runtime_generation(object).ok_or_else(|| { + tonic::Status::failed_precondition("sandbox lacks runtime generation") + })?; + let names = SandboxRuntimeNames::for_generation(&sandbox_id, generation); + let spec = object.data["spec"]["podTemplate"]["spec"].clone(); + let actual = crate::resource_admission::admit( + &self.client, + &self.config.resource_admission, + workspace, + namespace, + &spec, + &names.sandbox_secret, + ) + .await?; + if actual != expected { + return Err(tonic::Status::failed_precondition( + "sandbox external resource identity or attachment inventory changed", + )); + } + Ok(()) + } + pub async fn get_sandbox(&self, sandbox_id: &str) -> Result, String> { info!( sandbox_id = %sandbox_id, @@ -1673,6 +1822,11 @@ impl KubernetesComputeDriver { &self, sandbox: &Sandbox, ) -> Result { + openshell_core::resource_admission::check_sandbox_driver_config( + self.config.allow_driver_config, + sandbox, + ) + .map_err(admission_error)?; let gpu_requirements = sandbox .spec .as_ref() @@ -1693,7 +1847,8 @@ impl KubernetesComputeDriver { WorkspaceMode::Shared => self.config.namespace.clone(), WorkspaceMode::Managed => { let namespace = self.ensure_namespace(workspace).await?; - self.ensure_image_pull_secrets(&namespace).await?; + self.ensure_image_pull_secrets(&namespace, workspace) + .await?; namespace } WorkspaceMode::Operator => { @@ -1711,6 +1866,10 @@ impl KubernetesComputeDriver { if self.config.is_multi_namespace() { self.ensure_tls_secret(&target_namespace).await?; } + let resource_identities = self + .admit_requested_resources(sandbox) + .await + .map_err(admission_error)?; info!( sandbox_id = %sandbox.id, @@ -1772,6 +1931,21 @@ impl KubernetesComputeDriver { } let mut obj = DynamicObject::new(&kube_name, &agent_sandbox_api.resource); let mut annotations = sandbox_annotations(sandbox); + annotations.insert( + crate::resource_admission::IDENTITIES.into(), + serde_json::to_string(&resource_identities) + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?, + ); + annotations.insert( + crate::resource_admission::CONFIG_USED.into(), + sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()) + .and_then(|template| template.driver_config.as_ref()) + .is_some_and(|config| !config.fields.is_empty()) + .to_string(), + ); add_trace_context_annotation(&mut annotations); annotations.insert( ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING.to_string(), @@ -2284,6 +2458,14 @@ impl KubernetesComputeDriver { let workload_pod = self .wait_for_bootstrap_workload_pod(&pods, cr_name, cr_uid) .await?; + let admission_object = sandbox_api + .api + .get(cr_name) + .await + .map_err(KubernetesDriverError::from_kube)?; + self.admit_stored_resources(&admission_object) + .await + .map_err(admission_error)?; Self::validate_capability_free_workload_pod( &workload_pod, cr_uid, @@ -2549,6 +2731,14 @@ impl KubernetesComputeDriver { let workload_pod = self .wait_for_bootstrap_workload_pod(&pods, cr_name, cr_uid) .await?; + let admission_object = sandbox_api + .api + .get(cr_name) + .await + .map_err(KubernetesDriverError::from_kube)?; + self.admit_stored_resources(&admission_object) + .await + .map_err(admission_error)?; Self::validate_capability_free_workload_pod( &workload_pod, cr_uid, @@ -2865,6 +3055,9 @@ impl KubernetesComputeDriver { .map_err(KubernetesDriverError::from_kube)? .items; let mut object = select_expected_sandbox_runtime(objects, &expected_runtime_identity)?; + self.admit_stored_resources(&object) + .await + .map_err(admission_error)?; if sandbox_runtime_bootstrap_in_progress(&object) { let phase = sandbox_runtime_bootstrap_phase(&object); if phase != Some(SandboxRuntimeBootstrapPhase::Suspending) @@ -2917,6 +3110,9 @@ impl KubernetesComputeDriver { .map_err(KubernetesDriverError::from_kube)? .items; object = select_expected_sandbox_runtime(refreshed, &expected_runtime_identity)?; + self.admit_stored_resources(&object) + .await + .map_err(admission_error)?; } let namespace = object .metadata @@ -3497,6 +3693,14 @@ impl KubernetesComputeDriver { let Ok(sandbox_id) = sandbox_id_from_object(&object) else { continue; }; + if let Err(error) = self.admit_stored_resources(&object).await { + warn!(%sandbox_id, reason = %error.message(), "Sandbox resource admission revalidation failed"); + if error.code() == tonic::Code::FailedPrecondition { + self.suspend_sandbox_runtime_after_dependency_failure(&lookup_api, &object) + .await; + } + continue; + } let namespace = object .metadata .namespace @@ -4603,15 +4807,25 @@ fn managed_ssh_network_policy(namespace: &str, config: &KubernetesComputeConfig) } } -fn image_pull_secret_copy(secret_name: &str, namespace: &str, source: Secret) -> Secret { +fn image_pull_secret_copy( + secret_name: &str, + namespace: &str, + workspace: &str, + gateway_id: &str, + source: Secret, +) -> Secret { + let mut labels = source.metadata.labels.unwrap_or_default(); + labels.insert( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ); + labels.insert(LABEL_GATEWAY_ID.to_string(), gateway_id.to_string()); + labels.insert(LABEL_SANDBOX_WORKSPACE.to_string(), workspace.to_string()); Secret { metadata: ObjectMeta { name: Some(secret_name.to_string()), namespace: Some(namespace.to_string()), - labels: Some(BTreeMap::from([( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - )])), + labels: Some(labels), ..Default::default() }, data: source.data, @@ -4620,6 +4834,22 @@ fn image_pull_secret_copy(secret_name: &str, namespace: &str, source: Secret) -> } } +fn image_pull_secret_owned_by_gateway( + labels: Option<&BTreeMap>, + gateway_id: &str, + workspace: &str, +) -> bool { + labels.is_some_and(|labels| { + labels.get(LABEL_MANAGED_BY).map(String::as_str) == Some(LABEL_MANAGED_BY_VALUE) + && labels + .get(LABEL_GATEWAY_ID) + .is_none_or(|value| value == gateway_id) + && labels + .get(LABEL_SANDBOX_WORKSPACE) + .is_none_or(|value| value == workspace) + }) +} + fn sandbox_annotations(sandbox: &Sandbox) -> BTreeMap { let mut annotations = BTreeMap::new(); annotations.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); @@ -7056,6 +7286,64 @@ fn spawn_namespace_file_watcher( #[cfg(test)] mod tests { use super::*; + + #[tokio::test] + async fn admission_defaults_reject_driver_config_before_kubernetes_io_in_every_mode() { + for workspace_mode in [ + WorkspaceMode::Shared, + WorkspaceMode::Managed, + WorkspaceMode::Operator, + ] { + let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig { + workspace_mode, + ..Default::default() + }); + let sandbox = Sandbox { + workspace: "team-a".into(), + name: "attacker".into(), + spec: Some(SandboxSpec { + template: Some(SandboxTemplate { + driver_config: Some(json_struct( + serde_json::json!({"volumes":[{"name":"data","persistent_volume_claim":{"claim_name":"openshell-data-openshell-0"}}]}), + )), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + let error = driver.validate_sandbox_create(&sandbox).await.unwrap_err(); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert!(error.message().contains("allow_driver_config")); + assert!( + matches!(driver.create_sandbox_inner(&sandbox).await, Err(KubernetesDriverError::Precondition(message)) if message.contains("allow_driver_config")) + ); + } + } + + #[tokio::test] + async fn admission_allows_private_default_workload_without_external_lookups() { + let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig::default()); + let sandbox = Sandbox { + workspace: "team-a".into(), + name: "private".into(), + spec: Some(SandboxSpec { + template: Some(SandboxTemplate { + image: "test".into(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + assert!( + driver + .admit_requested_resources(&sandbox) + .await + .unwrap() + .is_empty() + ); + } use openshell_core::progress::{ PROGRESS_ACTIVE_DETAIL_KEY, PROGRESS_ACTIVE_STEP_KEY, PROGRESS_COMPLETE_LABEL_KEY, PROGRESS_COMPLETE_STEP_KEY, @@ -10361,7 +10649,7 @@ mod tests { } #[test] - fn image_pull_secret_copy_keeps_only_portable_secret_fields() { + fn image_pull_secret_copy_keeps_portable_fields_and_records_ownership() { let source: Secret = serde_json::from_value(serde_json::json!({ "apiVersion": "v1", "kind": "Secret", @@ -10379,7 +10667,7 @@ mod tests { })) .unwrap(); - let copy = image_pull_secret_copy("regcred", "workspace", source); + let copy = image_pull_secret_copy("regcred", "workspace", "team-a", "gateway-a", source); assert_eq!(copy.metadata.name.as_deref(), Some("regcred")); assert_eq!(copy.metadata.namespace.as_deref(), Some("workspace")); assert_eq!( @@ -10401,12 +10689,208 @@ mod tests { .map(String::as_str), Some(LABEL_MANAGED_BY_VALUE) ); + assert_eq!( + copy.metadata + .labels + .as_ref() + .unwrap() + .get(LABEL_GATEWAY_ID) + .map(String::as_str), + Some("gateway-a") + ); + assert_eq!( + copy.metadata + .labels + .as_ref() + .unwrap() + .get(LABEL_SANDBOX_WORKSPACE) + .map(String::as_str), + Some("team-a") + ); + assert_eq!( + copy.metadata + .labels + .as_ref() + .unwrap() + .get("source-only") + .map(String::as_str), + Some("true") + ); assert!(copy.metadata.uid.is_none()); assert!(copy.metadata.resource_version.is_none()); assert!(copy.metadata.annotations.is_none()); assert!(copy.metadata.finalizers.is_none()); } + #[test] + fn image_pull_secret_collision_requires_gateway_ownership() { + let legacy = BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )]); + assert!(image_pull_secret_owned_by_gateway( + Some(&legacy), + "gateway-a", + "team-a" + )); + + let owned = BTreeMap::from([ + ( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ), + (LABEL_GATEWAY_ID.to_string(), "gateway-a".to_string()), + (LABEL_SANDBOX_WORKSPACE.to_string(), "team-a".to_string()), + ]); + assert!(image_pull_secret_owned_by_gateway( + Some(&owned), + "gateway-a", + "team-a" + )); + assert!(!image_pull_secret_owned_by_gateway( + Some(&owned), + "gateway-b", + "team-a" + )); + assert!(!image_pull_secret_owned_by_gateway( + Some(&owned), + "gateway-a", + "team-b" + )); + assert!(!image_pull_secret_owned_by_gateway( + Some(&BTreeMap::new()), + "gateway-a", + "team-a" + )); + } + + #[tokio::test] + async fn managed_image_pull_secret_admits_source_before_copying() { + let source_path = "/api/v1/namespaces/openshell/secrets/regcred"; + let target_path = "/api/v1/namespaces/managed-team-a/secrets/regcred"; + let copied = serde_json::json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "regcred", + "namespace": "managed-team-a", + "labels": { + "openshell.ai/sandbox-attachable": "true", + LABEL_MANAGED_BY: LABEL_MANAGED_BY_VALUE, + LABEL_GATEWAY_ID: "gateway-a", + LABEL_SANDBOX_WORKSPACE: "team-a" + } + }, + "type": "kubernetes.io/dockerconfigjson", + "data": { ".dockerconfigjson": "e30=" } + }); + let steps = Arc::new(std::sync::Mutex::new(VecDeque::from([ + ( + http::Method::GET, + source_path, + kube_test_response( + http::StatusCode::OK, + serde_json::json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "regcred", + "namespace": "openshell", + "labels": {"openshell.ai/sandbox-attachable": "true"} + }, + "type": "kubernetes.io/dockerconfigjson", + "data": { ".dockerconfigjson": "e30=" } + }), + ), + ), + ( + http::Method::GET, + target_path, + kube_test_not_found("secrets", "regcred"), + ), + ( + http::Method::PATCH, + target_path, + kube_test_response(http::StatusCode::OK, copied), + ), + ]))); + let service_steps = steps.clone(); + let service = tower::service_fn(move |request: http::Request| { + let steps = service_steps.clone(); + async move { + let (method, path, response) = steps + .lock() + .unwrap() + .pop_front() + .expect("unexpected Kubernetes API request"); + assert_eq!(request.method(), method); + assert_eq!(request.uri().path(), path); + Ok::<_, std::convert::Infallible>(response) + } + }); + let client = Client::new(service, "openshell"); + let driver = KubernetesComputeDriver { + client: client.clone(), + watch_client: client, + sandbox_api_version: Arc::new(OnceCell::new()), + config: KubernetesComputeConfig { + namespace: "openshell".into(), + gateway_id: "gateway-a".into(), + image_pull_secrets: vec!["regcred".into()], + ..Default::default() + }, + operator_allowlist: None, + }; + + driver + .ensure_image_pull_secrets("managed-team-a", "team-a") + .await + .expect("approved source should be copied"); + assert!(steps.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn managed_image_pull_secret_rejects_unapproved_source_without_copying() { + let service = tower::service_fn( + move |request: http::Request| async move { + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().path(), + "/api/v1/namespaces/openshell/secrets/regcred" + ); + Ok::<_, std::convert::Infallible>(kube_test_response( + http::StatusCode::OK, + serde_json::json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": "regcred", "namespace": "openshell"}, + "type": "kubernetes.io/dockerconfigjson", + "data": { ".dockerconfigjson": "e30=" } + }), + )) + }, + ); + let client = Client::new(service, "openshell"); + let driver = KubernetesComputeDriver { + client: client.clone(), + watch_client: client, + sandbox_api_version: Arc::new(OnceCell::new()), + config: KubernetesComputeConfig { + namespace: "openshell".into(), + gateway_id: "gateway-a".into(), + image_pull_secrets: vec!["regcred".into()], + ..Default::default() + }, + operator_allowlist: None, + }; + + let error = driver + .ensure_image_pull_secrets("managed-team-a", "team-a") + .await + .expect_err("unapproved source must not be copied"); + assert!(matches!(error, KubernetesDriverError::Precondition(_))); + } + #[test] fn namespace_owned_with_correct_labels() { let labels = BTreeMap::from([ diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index bb861308a2..607f95d382 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -6,6 +6,7 @@ pub mod driver; pub mod grpc; pub mod isolation; pub mod otel_tracing; +mod resource_admission; mod sandbox_runtime; pub use config::{ diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index fbcaf1d5f6..0b6258082c 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -21,6 +21,13 @@ use openshell_driver_kubernetes::{ #[command(version = VERSION)] #[allow(clippy::struct_excessive_bools)] struct Args { + /// Operator-owned JSON policy; omitted means driver config disabled and labels required. + #[arg( + long, + env = "OPENSHELL_DRIVER_ADMISSION_CONFIG_JSON", + default_value = "{}" + )] + admission_config_json: openshell_core::resource_admission::DriverAdmissionConfig, /// Public compute-driver Unix socket used by an external gateway. #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] bind_socket: Option, @@ -231,6 +238,8 @@ async fn main() -> Result<()> { let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); let driver = KubernetesComputeDriver::new( KubernetesComputeConfig { + allow_driver_config: args.admission_config_json.allow_driver_config, + resource_admission: args.admission_config_json.resource_admission.clone(), workspace_mode: args.workspace_mode, gateway_id: args.gateway_id, namespace: args.sandbox_namespace, diff --git a/crates/openshell-driver-kubernetes/src/resource_admission.rs b/crates/openshell-driver-kubernetes/src/resource_admission.rs new file mode 100644 index 0000000000..26e5e0a95d --- /dev/null +++ b/crates/openshell-driver-kubernetes/src/resource_admission.rs @@ -0,0 +1,349 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Closed workload-reference inventory and metadata-only resource resolution. + +use kube::{ + Api, Client, + api::ApiResource, + core::{DynamicObject, GroupVersionKind}, +}; +use openshell_core::resource_admission::ResourceAdmissionConfig; +use serde_json::Value; +use std::collections::{BTreeMap, BTreeSet}; +use tonic::Status; + +pub const IDENTITIES: &str = "openshell.ai/resource-admission-identities"; +pub const CONFIG_USED: &str = "openshell.ai/caller-driver-config-used"; +pub type Identities = BTreeMap; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Scope { + /// Data-bearing resources selected for one `OpenShell` workspace. + Workspace, + /// Operator infrastructure intentionally reusable across workspaces. + Shared, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct Reference { + kind: &'static str, + name: String, + scope: Scope, +} + +fn reference(refs: &mut BTreeSet, kind: &'static str, name: Option<&str>, scope: Scope) { + if let Some(name) = name.filter(|name| !name.is_empty()) { + refs.insert(Reference { + kind, + name: name.into(), + scope, + }); + } +} + +fn inventory(spec: &Value, private_secret: &str) -> Result, Status> { + let deny = || { + Status::failed_precondition("workload contains an unsupported external resource attachment") + }; + let mut refs = BTreeSet::new(); + reference( + &mut refs, + "RuntimeClass", + spec["runtimeClassName"].as_str(), + Scope::Shared, + ); + reference( + &mut refs, + "PriorityClass", + spec["priorityClassName"].as_str(), + Scope::Shared, + ); + // With automatic and projected tokens prohibited, the workload gets no + // ServiceAccount credentials. Selecting the Pod's identity is not a grant. + if spec["automountServiceAccountToken"] != false + || spec + .get("resourceClaims") + .is_some_and(|v| v.as_array().is_none_or(|a| !a.is_empty())) + { + return Err(deny()); + } + for secret in spec["imagePullSecrets"].as_array().into_iter().flatten() { + reference(&mut refs, "Secret", secret["name"].as_str(), Scope::Shared); + } + for volume in spec["volumes"].as_array().into_iter().flatten() { + let object = volume.as_object().ok_or_else(deny)?; + let sources: Vec<_> = object.keys().filter(|key| key.as_str() != "name").collect(); + if sources.len() != 1 { + return Err(deny()); + } + match sources[0].as_str() { + "emptyDir" | "downwardAPI" => {} + "persistentVolumeClaim" => reference( + &mut refs, + "PersistentVolumeClaim", + volume["persistentVolumeClaim"]["claimName"].as_str(), + Scope::Workspace, + ), + "secret" => { + let name = volume["secret"]["secretName"].as_str(); + if name != Some(private_secret) { + reference(&mut refs, "Secret", name, Scope::Workspace); + } + } + "configMap" => reference( + &mut refs, + "ConfigMap", + volume["configMap"]["name"].as_str(), + Scope::Workspace, + ), + _ => return Err(deny()), + } + } + for field in ["containers", "initContainers", "ephemeralContainers"] { + for container in spec[field].as_array().into_iter().flatten() { + for env in container["envFrom"].as_array().into_iter().flatten() { + reference( + &mut refs, + "Secret", + env["secretRef"]["name"].as_str(), + Scope::Workspace, + ); + reference( + &mut refs, + "ConfigMap", + env["configMapRef"]["name"].as_str(), + Scope::Workspace, + ); + } + for env in container["env"].as_array().into_iter().flatten() { + reference( + &mut refs, + "Secret", + env["valueFrom"]["secretKeyRef"]["name"].as_str(), + Scope::Workspace, + ); + reference( + &mut refs, + "ConfigMap", + env["valueFrom"]["configMapKeyRef"]["name"].as_str(), + Scope::Workspace, + ); + } + for field in ["requests", "limits"] { + for (resource, _) in container["resources"][field] + .as_object() + .into_iter() + .flatten() + { + if resource.contains('/') && resource != "nvidia.com/gpu" { + return Err(deny()); + } + } + } + } + } + Ok(refs) +} + +/// Resolve references selected through the OpenShell-owned Pod template. +/// Kubernetes control-plane mutations of the eventual live Pod are outside the +/// workspace-user authorization boundary and are not inventoried here. +pub async fn admit( + client: &Client, + policy: &ResourceAdmissionConfig, + workspace: &str, + namespace: &str, + spec: &Value, + private_secret: &str, +) -> Result { + policy.validate().map_err(Status::failed_precondition)?; + if !policy.enabled { + return Ok(BTreeMap::new()); + } + let mut identities = BTreeMap::new(); + for reference in inventory(spec, private_secret)? { + let (group, version, plural, cluster) = match reference.kind { + "PersistentVolumeClaim" => ("", "v1", "persistentvolumeclaims", false), + "Secret" => ("", "v1", "secrets", false), + "ConfigMap" => ("", "v1", "configmaps", false), + "RuntimeClass" => ("node.k8s.io", "v1", "runtimeclasses", true), + "PriorityClass" => ("scheduling.k8s.io", "v1", "priorityclasses", true), + _ => unreachable!("closed resource inventory"), + }; + let resource = ApiResource::from_gvk_with_plural( + &GroupVersionKind::gvk(group, version, reference.kind), + plural, + ); + let api: Api = if cluster { + Api::all_with(client.clone(), &resource) + } else { + Api::namespaced_with(client.clone(), namespace, &resource) + }; + let object = tokio::time::timeout( + std::time::Duration::from_secs(30), + api.get_metadata(&reference.name), + ) + .await + .map_err(|_| Status::unavailable("resource admission lookup timed out"))? + .map_err(|error| match error { + kube::Error::Api(response) if response.code == 404 => { + Status::failed_precondition("external resource not admitted") + } + _ => Status::unavailable("resource admission metadata lookup failed"), + })?; + let metadata = object.metadata; + if metadata.deletion_timestamp.is_some() { + return Err(Status::failed_precondition("resource is being deleted")); + } + let uid = metadata + .uid + .filter(|uid| !uid.is_empty()) + .ok_or_else(|| Status::failed_precondition("resource has no identity"))?; + let labels = metadata + .labels + .as_ref() + .into_iter() + .flat_map(|labels| labels.iter()); + match reference.scope { + Scope::Workspace => policy.admit(workspace, labels)?, + Scope::Shared => policy.admit_shared(labels)?, + } + identities.insert( + format!( + "{}/{}/{}", + reference.kind, + if cluster { "" } else { namespace }, + reference.name + ), + uid, + ); + } + Ok(identities) +} + +pub fn check_record( + annotations: Option<&BTreeMap>, + allow_config: bool, +) -> Result { + let annotations = annotations.ok_or_else(|| { + Status::failed_precondition("sandbox lacks admission provenance; recreate it") + })?; + match annotations.get(CONFIG_USED).map(String::as_str) { + Some("false") => {} + Some("true") if allow_config => {} + _ => { + return Err(Status::failed_precondition( + "sandbox driver config is disabled or lacks admission provenance; recreate it", + )); + } + } + annotations + .get(IDENTITIES) + .ok_or_else(|| { + Status::failed_precondition("sandbox lacks resource identities; recreate it") + }) + .and_then(|value| { + serde_json::from_str(value) + .map_err(|_| Status::failed_precondition("invalid resource admission record")) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn pvc_admission_uses_resource_metadata_not_namespace_or_read_only() { + for (labels, allowed) in [ + (serde_json::json!({}), false), + ( + serde_json::json!({"openshell.ai/sandbox-attachable":"true","openshell.ai/workspace":"other"}), + false, + ), + ( + serde_json::json!({"openshell.ai/sandbox-attachable":"true","openshell.ai/workspace":"team-a"}), + true, + ), + ] { + for read_only in [false, true] { + let labels = labels.clone(); + let service = tower::service_fn( + move |request: http::Request| { + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().path(), + "/api/v1/namespaces/shared/persistentvolumeclaims/openshell-data-openshell-0" + ); + assert!( + request.headers()["accept"] + .to_str() + .unwrap() + .contains("PartialObjectMetadata") + ); + let body = serde_json::json!({"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadata", + "metadata":{"name":"openshell-data-openshell-0","namespace":"shared","uid":"fixture-pvc","labels":labels}}); + async move { + Ok::<_, std::convert::Infallible>( + http::Response::builder() + .header("content-type", "application/json") + .body(kube::client::Body::from(body.to_string().into_bytes())) + .unwrap(), + ) + } + }, + ); + let client = Client::new(service, "shared"); + let spec = serde_json::json!({"automountServiceAccountToken":false,"volumes":[{ + "name":"data","persistentVolumeClaim":{"claimName":"openshell-data-openshell-0","readOnly":read_only}}]}); + let result = admit( + &client, + &ResourceAdmissionConfig::default(), + "team-a", + "shared", + &spec, + "private", + ) + .await; + assert_eq!(result.is_ok(), allowed, "{result:?}"); + if let Ok(identities) = result { + assert_eq!(identities.len(), 1); + } + } + } + } + #[test] + fn inventories_all_containers_and_reference_aliases() { + let pod = serde_json::json!({"automountServiceAccountToken":false,"runtimeClassName":"r","priorityClassName":"p", + "volumes":[{"name":"data","persistentVolumeClaim":{"claimName":"gateway-db","readOnly":true}}], + "initContainers":[{"envFrom":[{"secretRef":{"name":"secret"}}]}], + "containers":[{"env":[{"valueFrom":{"configMapKeyRef":{"name":"config"}}}]}]}); + let refs = inventory(&pod, "private").unwrap(); + assert_eq!(refs.len(), 5); + assert!( + refs.iter() + .any(|r| r.name == "gateway-db" && r.scope == Scope::Workspace) + ); + assert!( + refs.iter() + .any(|r| r.name == "r" && r.scope == Scope::Shared) + ); + } + #[test] + fn rejects_unsupported_volume_sources_but_allows_gpu() { + for kind in ["hostPath", "csi", "projected", "image"] { + assert!(inventory(&serde_json::json!({"automountServiceAccountToken":false,"volumes":[{"name":"x",kind:{}}]}), "private").is_err()); + } + assert!(inventory(&serde_json::json!({"automountServiceAccountToken":false,"containers":[{"resources":{"limits":{"nvidia.com/gpu":"1"}}}]}), "private").is_ok()); + } + #[test] + fn legacy_and_forbidden_config_records_fail_closed() { + assert!(check_record(None, true).is_err()); + let record = BTreeMap::from([ + (CONFIG_USED.into(), "true".into()), + (IDENTITIES.into(), "{}".into()), + ]); + assert!(check_record(Some(&record), false).is_err()); + assert!(check_record(Some(&record), true).is_ok()); + } +} diff --git a/crates/openshell-driver-mxc/README.md b/crates/openshell-driver-mxc/README.md index 9c235b85c7..b4ee82abb2 100644 --- a/crates/openshell-driver-mxc/README.md +++ b/crates/openshell-driver-mxc/README.md @@ -4,6 +4,12 @@ OpenShell compute driver backed by **Microsoft MXC** (`wxc-exec`) on Windows. ## Design +Caller driver config is disabled by default, so command-based MXC workflows +need explicit administrator opt-in. Host filesystem grants have no trusted +label resolver and are rejected while resource admission is enabled. +See [resource admission configuration](../../docs/reference/gateway-config.mdx#external-resource-admission) +for the independent controls and the security consequences of opting out. + This driver implements the gateway's ordinary in-process `ComputeDriver` contract and is linked into `openshell-gateway`. It sets `driver_reports_runtime_readiness`, so the gateway accepts driver-reported diff --git a/crates/openshell-driver-mxc/src/driver.rs b/crates/openshell-driver-mxc/src/driver.rs index 23491533bb..31692d659f 100644 --- a/crates/openshell-driver-mxc/src/driver.rs +++ b/crates/openshell-driver-mxc/src/driver.rs @@ -59,6 +59,10 @@ pub enum MxcBackend { // configuration schema rather than one compound state machine. #[allow(clippy::struct_excessive_bools)] pub struct MxcComputeConfig { + /// Permit caller-supplied driver JSON. Does not waive resource admission. + pub allow_driver_config: bool, + /// Operator-owned external attachment approval policy. + pub resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig, /// Path to `wxc-exec.exe`. Required for live runs. pub wxc_exec_path: String, /// Backend to target. Default: `process_container`. @@ -91,6 +95,9 @@ impl Default for MxcComputeConfig { fn default() -> Self { Self { wxc_exec_path: "wxc-exec.exe".into(), + allow_driver_config: false, + resource_admission: + openshell_core::resource_admission::ResourceAdmissionConfig::default(), backend: MxcBackend::default(), pc_least_privilege: false, pc_capabilities: Vec::new(), @@ -453,6 +460,11 @@ impl MxcComputeBackend { pub fn capabilities(&self) -> GetCapabilitiesResponse { GetCapabilitiesResponse { + resource_admission_policy: openshell_core::resource_admission::DriverAdmissionConfig { + allow_driver_config: self.config.allow_driver_config, + resource_admission: self.config.resource_admission.clone(), + } + .acknowledgement(), driver_name: DRIVER_NAME.to_string(), driver_version: DRIVER_VERSION.to_string(), default_image: DEFAULT_IMAGE_SENTINEL.to_string(), @@ -513,6 +525,19 @@ impl MxcComputeBackend { } pub fn validate_sandbox_create(&self, sandbox: &DriverSandbox) -> Result<(), tonic::Status> { + self.config + .resource_admission + .validate() + .map_err(tonic::Status::failed_precondition)?; + openshell_core::resource_admission::check_sandbox_driver_config( + self.config.allow_driver_config, + sandbox, + )?; + // MXC grants access to existing host filesystem objects, including its + // executable/workdir. There is no authoritative label resolver yet. + self.config + .resource_admission + .reject_unlabelable("MXC host filesystem grants")?; Self::validate_sandbox_fields(sandbox)?; let policy = sandbox.spec.as_ref().and_then(|spec| spec.policy.as_ref()); let egress_addr = configured_egress_addr(&self.config)?; @@ -533,6 +558,7 @@ impl MxcComputeBackend { } pub async fn create_sandbox(&self, sandbox: &DriverSandbox) -> Result<(), tonic::Status> { + self.validate_sandbox_create(sandbox)?; let sandbox_id = sandbox.id.clone(); Self::validate_sandbox_fields(sandbox)?; @@ -1232,6 +1258,30 @@ mod lifecycle_tests { }; use std::time::Duration; + fn host_grants_config() -> MxcComputeConfig { + MxcComputeConfig { + allow_driver_config: true, + resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig { + enabled: false, + ..Default::default() + }, + ..Default::default() + } + } + + #[tokio::test] + async fn admission_rejects_host_grants_even_with_driver_config_enabled() { + let backend = MxcComputeBackend::new_mocked(MxcComputeConfig { + allow_driver_config: true, + ..Default::default() + }); + let sandbox = driver_sandbox("sb-admission"); + let error = backend.validate_sandbox_create(&sandbox).unwrap_err(); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert!(backend.create_sandbox(&sandbox).await.is_err()); + assert!(backend.get_sandbox("sb-admission").await.is_none()); + } + fn driver_sandbox(id: &str) -> DriverSandbox { driver_sandbox_with_command(id, "", vec!["cmd".into(), "/c".into(), "exit 0".into()]) } @@ -1422,7 +1472,7 @@ mod lifecycle_tests { "-Command".into(), format!("Set-Content -LiteralPath {hello} -Value hi"), ]; - let backend = MxcComputeBackend::new_mocked(MxcComputeConfig::default()); + let backend = MxcComputeBackend::new_mocked(host_grants_config()); let policy = fs_policy(&[&share]); let sb = with_policy(driver_sandbox_with_command("sb-pos", &share, cmd), policy); @@ -1483,7 +1533,7 @@ mod lifecycle_tests { "-Command".into(), format!("Set-Content -LiteralPath {hello} -Value hi"), ]; - let backend = MxcComputeBackend::new_mocked(MxcComputeConfig::default()); + let backend = MxcComputeBackend::new_mocked(host_grants_config()); let policy = fs_policy(&[&share]); let sb = with_policy(driver_sandbox_with_command("sb-pc", &share, cmd), policy); @@ -1535,7 +1585,7 @@ mod lifecycle_tests { backend: MxcBackend::ProcessContainer, egress_proxy: true, egress_proxy_addr: "127.0.0.1:18080".into(), - ..Default::default() + ..host_grants_config() }; let backend = MxcComputeBackend::new_mocked(config); let mut stream = backend.watch_sandboxes().await; @@ -1644,7 +1694,7 @@ mod lifecycle_tests { "-Command".into(), format!("Set-Content -LiteralPath {out_path} -Value hi"), ]; - let backend = MxcComputeBackend::new_mocked(MxcComputeConfig::default()); + let backend = MxcComputeBackend::new_mocked(host_grants_config()); // Subscribe to the watch stream BEFORE create so we catch the denial event. let mut stream = backend.watch_sandboxes().await; @@ -1703,7 +1753,7 @@ mod lifecycle_tests { "-Command".into(), format!("$null = '{share}'; Start-Sleep -Seconds 60"), ]; - let backend = MxcComputeBackend::new_mocked(MxcComputeConfig::default()); + let backend = MxcComputeBackend::new_mocked(host_grants_config()); let policy = fs_policy(&[&share]); let sandbox = with_policy(driver_sandbox_with_command("sb-stop", "", command), policy); backend @@ -1733,7 +1783,7 @@ mod lifecycle_tests { let tmp = tempfile::tempdir().unwrap(); let share = tmp.path().to_string_lossy().replace('\\', "/"); - let backend = MxcComputeBackend::new_mocked(MxcComputeConfig::default()); + let backend = MxcComputeBackend::new_mocked(host_grants_config()); let mut policy = fs_policy(&[&share]); policy.network_policies.insert( @@ -1761,7 +1811,7 @@ mod lifecycle_tests { let config = MxcComputeConfig { egress_proxy: true, egress_proxy_addr: "127.0.0.1:18080".into(), - ..Default::default() + ..host_grants_config() }; let backend = MxcComputeBackend::new_mocked(config); diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 85cdb33259..5147d78a51 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -13,6 +13,13 @@ identity, DNS, TCP, and loopback-forwarding semantics. ## Runtime posture +Caller driver config is disabled by default. Existing volumes require +administrator-controlled approval labels; bind and supplemental image mounts +are denied under enforcement. Private-volume names alone do not prove +ownership. GPU devices are temporarily exempt. Admission runs before launch, +restart, and periodically for running workloads. +See [resource admission configuration](../../docs/reference/gateway-config.mdx#external-resource-admission). + | Property | Workload | Supervisor | |---|---|---| | UID/GID | Pinned non-root workload identity | Same mapped identity | @@ -107,7 +114,9 @@ requires the authenticated supervisor session before publishing Ready. User `bind`, `volume`, `tmpfs`, and `image` mounts and CDI GPU selection remain native Podman features and apply only to the workload. Bind mounts require the -operator's `enable_bind_mounts` opt-in. Reserved control paths and the workspace +operator's `enable_bind_mounts` opt-in and disabled label admission. Supplemental +image mounts also require disabled admission. Driver JSON requires +`allow_driver_config = true`. Reserved control paths and the workspace root cannot be replaced. User-owned volumes are never created or deleted. See [gateway configuration](../../docs/reference/gateway-config.mdx) for diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 8bf332d296..80e24513a3 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -88,6 +88,8 @@ pub fn validate_name(name: &str) -> Result<(), PodmanApiError> { #[derive(Debug, Clone, serde::Deserialize)] #[serde(rename_all = "PascalCase")] pub struct ContainerInspect { + #[serde(default)] + pub mounts: Option>, pub id: String, pub name: String, pub state: ContainerState, @@ -223,12 +225,24 @@ pub struct PortMappingEntry { #[derive(Debug, Clone, Default, serde::Deserialize)] #[serde(rename_all = "PascalCase")] pub struct VolumeInspect { + #[serde(default)] + pub created_at: Option, + #[serde(default)] + pub labels: Option>, + #[serde(default)] + pub name: String, #[serde(default)] pub driver: String, #[serde(default)] pub options: HashMap, } +impl VolumeInspect { + pub(crate) fn admission_identity(&self) -> Value { + serde_json::json!({"name": self.name, "driver": self.driver, "options": self.options, "created_at": self.created_at}) + } +} + /// A Podman event from the events stream. #[derive(Debug, Clone, serde::Deserialize)] #[serde(rename_all = "PascalCase")] @@ -686,11 +700,53 @@ impl PodmanClient { // ── Volume operations ──────────────────────────────────────────────── - /// Create a named volume. Idempotent (conflict is ignored). - pub async fn create_volume(&self, name: &str) -> Result<(), PodmanApiError> { - validate_name(name)?; - self.create_ignore_conflict("/libpod/volumes/create", &serde_json::json!({"Name": name})) - .await + /// Never adopt an unrelated existing volume on a private provisioning path. + pub(crate) async fn create_owned_volume( + &self, + name: &str, + sandbox_id: &str, + workspace: &str, + ) -> Result<(), PodmanApiError> { + let labels = HashMap::from([ + ( + openshell_core::driver_utils::LABEL_SANDBOX_ID.to_string(), + sandbox_id.to_string(), + ), + ( + openshell_core::driver_utils::LABEL_SANDBOX_WORKSPACE.to_string(), + workspace.to_string(), + ), + ]); + match self.inspect_volume(name).await { + Ok(existing) => { + if existing.driver != "local" + || !existing.options.is_empty() + || existing.labels.as_ref() != Some(&labels) + { + return Err(PodmanApiError::InvalidInput( + "private volume name collides with an unrelated resource".into(), + )); + } + return Ok(()); + } + Err(PodmanApiError::NotFound(_)) => {} + Err(error) => return Err(error), + } + self.create_ignore_conflict( + "/libpod/volumes/create", + &serde_json::json!({"Name":name,"Driver":"local","Labels":labels}), + ) + .await?; + let created = self.inspect_volume(name).await?; + if created.driver != "local" + || !created.options.is_empty() + || created.labels.as_ref() != Some(&labels) + { + return Err(PodmanApiError::InvalidInput( + "private volume ownership verification failed".into(), + )); + } + Ok(()) } /// Remove a named volume. Idempotent (not-found is ignored). diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 3812367234..75dc2aa77e 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -27,6 +27,10 @@ pub const fn podman_image_pull_policy(policy: ImagePullPolicy) -> &'static str { #[derive(Clone, serde::Serialize, serde::Deserialize)] #[serde(default, deny_unknown_fields)] pub struct PodmanComputeConfig { + /// Permit caller-supplied driver JSON. Does not waive resource admission. + pub allow_driver_config: bool, + /// Operator-owned external attachment approval policy. + pub resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig, /// Podman API Unix socket. When unset, use the socket selected by /// gateway auto-detection. pub socket_path: Option, @@ -224,6 +228,9 @@ pub fn parse_id_map_entry( impl PodmanComputeConfig { /// Validate and normalize startup configuration without connecting to Podman. pub fn validate_configuration(&mut self) -> Result<(), crate::client::PodmanApiError> { + self.resource_admission + .validate() + .map_err(crate::client::PodmanApiError::InvalidInput)?; self.validate_tls_config()?; self.validate_runtime_limits()?; self.validate_host_gateway_ip()?; @@ -462,6 +469,9 @@ impl Default for PodmanComputeConfig { fn default() -> Self { Self { socket_path: None, + allow_driver_config: false, + resource_admission: + openshell_core::resource_admission::ResourceAdmissionConfig::default(), default_image: openshell_core::image::default_sandbox_image(), image_pull_policy: ImagePullPolicy::default(), grpc_endpoint: String::new(), diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index e25c30a1fb..1c7d0f62db 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -85,6 +85,23 @@ pub struct PodmanSandboxDriverConfig { } impl PodmanSandboxDriverConfig { + pub(crate) fn admit_mount_types( + &self, + policy: &openshell_core::resource_admission::ResourceAdmissionConfig, + ) -> Result<(), ComputeDriverError> { + for mount in &self.mounts { + if matches!( + mount, + PodmanDriverMountConfig::Bind { .. } | PodmanDriverMountConfig::Image { .. } + ) { + policy + .reject_unlabelable("host bind or image mount") + .map_err(|error| ComputeDriverError::Precondition(error.message().into()))?; + } + } + Ok(()) + } + pub fn from_sandbox(sandbox: &DriverSandbox) -> Result { let Some(template) = sandbox .spec @@ -653,6 +670,13 @@ fn build_labels(sandbox: &DriverSandbox) -> BTreeMap { } } // Managed labels (highest priority -- always overwrite). + labels.insert( + openshell_core::resource_admission::CONFIG_USED_LABEL.into(), + template + .and_then(|t| t.driver_config.as_ref()) + .is_some_and(|config| !config.fields.is_empty()) + .to_string(), + ); labels.insert(LABEL_SANDBOX_ID.into(), sandbox.id.clone()); labels.insert(LABEL_SANDBOX_NAME.into(), sandbox.name.clone()); labels.insert(LABEL_SANDBOX_NAMESPACE.into(), sandbox.namespace.clone()); @@ -1083,7 +1107,13 @@ fn build_base_spec( let vol = volume_name(&sandbox.id); let env = build_env(sandbox, config, requested_image, oci_user)?; - let labels = build_labels(sandbox); + let mut labels = build_labels(sandbox); + labels.insert( + "openshell.ai/runtime-binary-source".into(), + supervisor_bin_path + .map(|path| path.display().to_string()) + .unwrap_or_default(), + ); let resource_limits = build_resource_limits(sandbox, config); let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts) .map_err(ComputeDriverError::InvalidArgument)?; @@ -1411,6 +1441,20 @@ pub struct IsolationSpecs { pub supervisor: ContainerSpec, } +impl IsolationSpecs { + pub(crate) fn record_resource_identities( + &mut self, + identities: &BTreeMap, + ) -> Result<(), ComputeDriverError> { + self.workload.labels.insert( + openshell_core::resource_admission::IDENTITIES_LABEL.into(), + serde_json::to_string(identities) + .map_err(|error| ComputeDriverError::Message(error.to_string()))?, + ); + Ok(()) + } +} + pub fn build_isolation_specs( input: IsolationSpecInput<'_>, ) -> Result { diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 2c9303f1c1..d366ebe00e 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -507,7 +507,7 @@ impl PodmanComputeDriver { ); } - Ok(Self { + let driver = Self { client, config, rootless, @@ -517,12 +517,25 @@ impl PodmanComputeDriver { )), gpu_inventory_refresh: Arc::new(local_podman_gpu_selector_state), lifecycle_event_fences: LifecycleEventFences::default(), - }) + }; + let reconciler = driver.clone(); + tokio::spawn(async move { + loop { + reconciler.reconcile_resource_admission().await; + tokio::time::sleep(Duration::from_secs(30)).await; + } + }); + Ok(driver) } /// Report driver capabilities. pub fn capabilities(&self) -> Result { Ok(GetCapabilitiesResponse { + resource_admission_policy: openshell_core::resource_admission::DriverAdmissionConfig { + allow_driver_config: self.config.allow_driver_config, + resource_admission: self.config.resource_admission.clone(), + } + .acknowledgement(), driver_name: "podman".to_string(), driver_version: openshell_core::VERSION.to_string(), default_image: self.config.default_image.clone(), @@ -570,12 +583,18 @@ impl PodmanComputeDriver { &self, sandbox: &'a DriverSandbox, ) -> Result, ComputeDriverError> { + openshell_core::resource_admission::check_sandbox_driver_config( + self.config.allow_driver_config, + sandbox, + ) + .map_err(|error| ComputeDriverError::Precondition(error.message().into()))?; let gpu_requirements = sandbox .spec .as_ref() .and_then(|spec| spec.resource_requirements.as_ref()) .and_then(|requirements| driver_gpu_requirements(Some(requirements))); let driver_config = PodmanSandboxDriverConfig::from_sandbox(sandbox)?; + driver_config.admit_mount_types(&self.config.resource_admission)?; Self::validate_gpu_request(gpu_requirements, &driver_config)?; self.validate_user_volume_mounts_available(sandbox).await?; let _ = self.resolve_gpu_cdi_devices( @@ -646,13 +665,28 @@ impl PodmanComputeDriver { async fn validate_user_volume_mounts_available( &self, sandbox: &DriverSandbox, - ) -> Result<(), ComputeDriverError> { + ) -> Result, ComputeDriverError> { + let mut identities = std::collections::BTreeMap::new(); let volumes = container::podman_driver_volume_mount_sources(sandbox, self.config.enable_bind_mounts) .map_err(ComputeDriverError::Precondition)?; for volume in volumes { match self.client.inspect_volume(&volume).await { Ok(volume_info) => { + identities.insert(volume.clone(), volume_info.admission_identity()); + self.config + .resource_admission + .admit( + &sandbox.workspace, + volume_info + .labels + .as_ref() + .into_iter() + .flat_map(|labels| labels.iter()), + ) + .map_err(|error| { + ComputeDriverError::Precondition(error.message().into()) + })?; if !self.config.enable_bind_mounts && podman_volume_is_bind_backed(&volume_info) { return Err(ComputeDriverError::Precondition(format!( @@ -668,9 +702,139 @@ impl PodmanComputeDriver { Err(err) => return Err(ComputeDriverError::from(err)), } } + Ok(identities) + } + + /// Create a sandbox container. + async fn admit_container_resources(&self, id: &str) -> Result<(), ComputeDriverError> { + if self.config.allow_driver_config && !self.config.resource_admission.enabled { + return Ok(()); + } + let inspect = self.client.inspect_container(id).await?; + let labels = &inspect.config.labels; + let precondition = + |error: tonic::Status| ComputeDriverError::Precondition(error.message().into()); + openshell_core::resource_admission::check_config_provenance( + self.config.allow_driver_config, + labels + .get(openshell_core::resource_admission::CONFIG_USED_LABEL) + .map(String::as_str), + ) + .map_err(precondition)?; + if !self.config.resource_admission.enabled { + return Ok(()); + } + let missing = + || ComputeDriverError::Precondition("sandbox lacks attachment provenance".into()); + let workspace = labels + .get(container::LABEL_SANDBOX_WORKSPACE) + .ok_or_else(missing)?; + let sandbox_id = labels.get(LABEL_SANDBOX_ID).ok_or_else(missing)?; + let mounts = inspect.mounts.as_ref().ok_or_else(missing)?; + let expected: std::collections::BTreeMap = labels + .get(openshell_core::resource_admission::IDENTITIES_LABEL) + .and_then(|value| serde_json::from_str(value).ok()) + .ok_or_else(missing)?; + let mut actual = std::collections::BTreeMap::new(); + for mount in mounts { + match mount["Type"].as_str() { + Some("volume") => { + let name = mount["Name"].as_str().ok_or_else(missing)?; + let volume = + self.client + .inspect_volume(name) + .await + .map_err(|error| match error { + PodmanApiError::NotFound(_) => ComputeDriverError::Precondition( + "attached volume no longer exists".into(), + ), + other => ComputeDriverError::from(other), + })?; + if volume.name != name { + return Err(missing()); + } + if name == container::volume_name(sandbox_id) + || name == crate::isolation::channel_volume_name(sandbox_id) + { + let owned = volume.labels.as_ref().is_some_and(|labels| { + labels.get(LABEL_SANDBOX_ID) == Some(sandbox_id) + && labels.get(container::LABEL_SANDBOX_WORKSPACE) == Some(workspace) + }); + if !owned || volume.driver != "local" || !volume.options.is_empty() { + return Err(missing()); + } + } else { + actual.insert(name.to_string(), volume.admission_identity()); + self.config + .resource_admission + .admit( + workspace, + volume + .labels + .as_ref() + .into_iter() + .flat_map(|labels| labels.iter()), + ) + .map_err(precondition)?; + if !self.config.enable_bind_mounts && podman_volume_is_bind_backed(&volume) + { + return Err(ComputeDriverError::Precondition( + "bind-backed volume is disabled".into(), + )); + } + } + } + Some("tmpfs") => {} + Some("bind") + if mount["Destination"].as_str() + == Some(openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY) + && mount["RW"].as_bool() == Some(false) + && labels + .get("openshell.ai/runtime-binary-source") + .filter(|path| !path.is_empty()) + .map(String::as_str) + == mount["Source"].as_str() => {} + _ => self + .config + .resource_admission + .reject_unlabelable("effective Podman mount") + .map_err(precondition)?, + } + } + if actual != expected { + return Err(ComputeDriverError::Precondition( + "external volume identity or attachment inventory changed".into(), + )); + } Ok(()) } + /// Revalidate running grants every 30 seconds. Outages block launches but + /// only confirmed denials stop existing workloads. + async fn reconcile_resource_admission(&self) { + let Ok(entries) = self + .client + .list_containers(&[LABEL_MANAGED_FILTER, crate::isolation::WORKLOAD_FILTER]) + .await + else { + return; + }; + for entry in entries.iter().filter(|entry| entry.state == "running") { + if let Err(ComputeDriverError::Precondition(reason)) = + self.admit_container_resources(&entry.id).await + { + warn!(container = %entry.id, %reason, "Stopping sandbox after resource admission denial"); + let _ = self.client.stop_container(&entry.id, 0).await; + if let Some(id) = entry.labels.get(LABEL_SANDBOX_ID) { + let _ = self + .client + .stop_container(&crate::isolation::supervisor_name(id), 0) + .await; + } + } + } + } + /// Create a sandbox container. #[tracing::instrument( name = "podman.provision", @@ -837,7 +1001,7 @@ impl PodmanComputeDriver { let phase_status = openshell_otel::ErrorStatusGuard::current(); let result = async { self.client - .create_volume(&vol_name) + .create_owned_volume(&vol_name, &sandbox.id, &sandbox.workspace) .await .map_err(ComputeDriverError::from)?; let token_secret_name = @@ -876,8 +1040,11 @@ impl PodmanComputeDriver { // Clean up the volume and both per-sandbox secrets on any failure past // this point. + let channel_owned = std::sync::atomic::AtomicBool::new(false); let cleanup_created = || async { - let _ = self.client.remove_volume(&channel_volume).await; + if channel_owned.load(std::sync::atomic::Ordering::Relaxed) { + let _ = self.client.remove_volume(&channel_volume).await; + } let _ = self.client.remove_volume(&vol_name).await; if let Some(secret) = token_secret_name.as_deref() { cleanup_sandbox_token_secret(&self.client, secret).await; @@ -946,7 +1113,7 @@ impl PodmanComputeDriver { identity: &identity, rootless: self.rootless, }); - let specs = match specs { + let mut specs = match specs { Ok(spec) => spec, Err(e) => { cleanup_all().await; @@ -956,10 +1123,16 @@ impl PodmanComputeDriver { let mut created_workload = None; let mut created_supervisor = None; let create_result = async { - self.client.create_volume(&channel_volume).await?; + let identities = self.validate_user_volume_mounts_available(sandbox).await?; + specs.record_resource_identities(&identities)?; + self.client + .create_owned_volume(&channel_volume, &sandbox.id, &sandbox.workspace) + .await?; + channel_owned.store(true, std::sync::atomic::Ordering::Relaxed); let workload_id = self.client.create_typed_container(&specs.workload).await?; created_workload = Some(workload_id.clone()); self.client.verify_isolation_fence(&workload_id).await?; + self.admit_container_resources(&workload_id).await?; let child_env = podman_child_environment(sandbox, &image_env); let launch_authentication = sandbox .spec @@ -1249,6 +1422,7 @@ impl PodmanComputeDriver { .find_container(sandbox_id) .await? .ok_or(ComputeDriverError::NotFound)?; + self.admit_container_resources(&container.id).await?; if container.state == "running" { let supervisor = self .client @@ -2181,7 +2355,7 @@ mod tests { let _tracing_lock = openshell_otel_test_support::tracing_test_lock().await; let (socket_path, requests, handle) = spawn_podman_stub( "trace-create", - create_setup_responses(false) + create_setup_responses(false, "sandbox-trace") .into_iter() .chain(create_launch_responses()) .collect(), @@ -2614,7 +2788,10 @@ mod tests { async fn validate_sandbox_create_passes_explicit_cdi_device_id_without_inventory() { use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate}; - let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig::default()); + let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + allow_driver_config: true, + ..Default::default() + }); let sandbox = DriverSandbox { spec: Some(DriverSandboxSpec { resource_requirements: Some(gpu_resources(None)), @@ -2721,6 +2898,11 @@ mod tests { fn test_driver(socket_path: PathBuf) -> PodmanComputeDriver { let config = PodmanComputeConfig { + allow_driver_config: true, + resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig { + enabled: false, + ..Default::default() + }, socket_path: Some(socket_path), stop_timeout_secs: 10, ..PodmanComputeConfig::default() @@ -2728,7 +2910,9 @@ mod tests { PodmanComputeDriver::for_tests(config) } - fn test_driver_with_config(config: PodmanComputeConfig) -> PodmanComputeDriver { + fn test_driver_with_config(mut config: PodmanComputeConfig) -> PodmanComputeDriver { + config.allow_driver_config = true; + config.resource_admission.enabled = false; PodmanComputeDriver::for_tests(config) } @@ -2763,6 +2947,90 @@ mod tests { } } + #[tokio::test] + async fn private_volume_collision_is_not_adopted_or_relabelled() { + let (socket, requests, handle) = spawn_podman_stub( + "admission-collision", + vec![StubResponse::new( + StatusCode::OK, + serde_json::json!({ + "Name":"private-collision", "Driver":"local", "Options":{}, "Labels":{} + }) + .to_string(), + )], + ); + let driver = test_driver(socket.clone()); + assert!( + driver + .client + .create_owned_volume("private-collision", "sandbox-123", "team-a") + .await + .is_err() + ); + handle.await.unwrap(); + let logged = requests.lock().unwrap(); + assert_eq!(logged.len(), 1); + assert!(logged[0].starts_with("GET ")); + let _ = fs::remove_file(socket); + } + + #[tokio::test] + async fn admission_requires_explicit_volume_labels_and_workspace_match() { + for (labels, allowed) in [ + (serde_json::json!(null), false), + (serde_json::json!({}), false), + ( + serde_json::json!({"openshell.ai/sandbox-attachable":"true","openshell.ai/workspace":"other"}), + false, + ), + ( + serde_json::json!({"openshell.ai/sandbox-attachable":"true","openshell.ai/workspace":"team-a"}), + true, + ), + ] { + let (socket, requests, handle) = spawn_podman_stub("admission-labels", vec![StubResponse::new(StatusCode::OK, + serde_json::json!({"Name":"existing","Driver":"local","Options":{},"Labels":labels}).to_string())]); + let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + socket_path: Some(socket.clone()), + allow_driver_config: true, + ..Default::default() + }); + let mut sandbox = sandbox_with_volume_mount("existing"); + sandbox.workspace = "team-a".into(); + assert_eq!( + driver.validate_sandbox_create(&sandbox).await.is_ok(), + allowed + ); + handle.await.unwrap(); + assert!( + requests + .lock() + .unwrap() + .iter() + .all(|request| request.starts_with("GET ")) + ); + let _ = fs::remove_file(socket); + } + } + + #[tokio::test] + async fn admission_driver_config_denial_does_not_contact_podman() { + for enabled in [true, false] { + let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig { + resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig { + enabled, + ..Default::default() + }, + ..Default::default() + }); + let error = driver + .validate_sandbox_create(&sandbox_with_volume_mount("existing")) + .await + .unwrap_err(); + assert!(error.to_string().contains("allow_driver_config")); + } + } + fn api_path(path: &str) -> String { format!("/v5.0.0{path}") } @@ -2770,6 +3038,9 @@ mod tests { #[test] fn podman_local_volume_with_bind_option_is_bind_backed() { let volume = VolumeInspect { + created_at: None, + labels: None, + name: String::new(), driver: "local".to_string(), options: HashMap::from([("o".to_string(), "rw,bind".to_string())]), }; @@ -2780,6 +3051,9 @@ mod tests { #[test] fn podman_local_volume_with_rbind_option_is_bind_backed() { let volume = VolumeInspect { + created_at: None, + labels: None, + name: String::new(), driver: "local".to_string(), options: HashMap::from([("o".to_string(), "rw,rbind".to_string())]), }; @@ -2790,6 +3064,9 @@ mod tests { #[test] fn podman_empty_driver_volume_with_bind_option_is_bind_backed() { let volume = VolumeInspect { + created_at: None, + labels: None, + name: String::new(), driver: String::new(), options: HashMap::from([("o".to_string(), "bind".to_string())]), }; @@ -2800,6 +3077,9 @@ mod tests { #[test] fn podman_local_volume_without_bind_option_is_not_bind_backed() { let volume = VolumeInspect { + created_at: None, + labels: None, + name: String::new(), driver: "local".to_string(), options: HashMap::from([("o".to_string(), "addr=127.0.0.1,rw".to_string())]), }; @@ -2810,6 +3090,9 @@ mod tests { #[test] fn podman_nonlocal_volume_with_bind_option_is_not_bind_backed() { let volume = VolumeInspect { + created_at: None, + labels: None, + name: String::new(), driver: "custom".to_string(), options: HashMap::from([("o".to_string(), "bind".to_string())]), }; @@ -3164,7 +3447,7 @@ mod tests { StubResponse::new(StatusCode::OK, archive) } - fn create_setup_responses(proxy_secret: bool) -> Vec { + fn create_setup_responses(proxy_secret: bool, sandbox_id: &str) -> Vec { let mut responses = vec![ StubResponse::new(StatusCode::OK, "{}"), // sandbox runtime pull StubResponse::new(StatusCode::OK, "{}"), // supervisor pull @@ -3177,7 +3460,9 @@ mod tests { StubResponse::new(StatusCode::NO_CONTENT, ""), // remove stopped reader image_response("sha256:sandbox-runtime"), image_response("sha256:supervisor"), + StubResponse::new(StatusCode::NOT_FOUND, ""), // no existing private workspace StubResponse::new(StatusCode::CREATED, "{}"), // workspace volume + owned_volume_response(&container::volume_name(sandbox_id), sandbox_id), ]; if proxy_secret { responses.push(StubResponse::new(StatusCode::CREATED, "{}")); @@ -3188,10 +3473,26 @@ mod tests { sandbox_binary_archive_response(), StubResponse::new(StatusCode::NO_CONTENT, ""), // remove extractor ]); + responses.push(StubResponse::new(StatusCode::NOT_FOUND, "")); responses.push(StubResponse::new(StatusCode::CREATED, "{}")); // channel volume + responses.push(owned_volume_response( + &crate::isolation::channel_volume_name(sandbox_id), + sandbox_id, + )); responses } + fn owned_volume_response(name: &str, sandbox_id: &str) -> StubResponse { + StubResponse::new( + StatusCode::OK, + serde_json::json!({ + "Name": name, "Driver": "local", "Options": {}, + "Labels": {LABEL_SANDBOX_ID: sandbox_id, container::LABEL_SANDBOX_WORKSPACE: ""} + }) + .to_string(), + ) + } + fn create_launch_responses() -> Vec { vec![ created_response("workload"), @@ -3254,7 +3555,7 @@ mod tests { let auth_file = write_proxy_auth_file("create-fail"); let (socket_path, request_log, handle) = spawn_podman_stub( "create-container-fail", - create_setup_responses(true) + create_setup_responses(true, sandbox_id) .into_iter() .chain([ StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, "create failed"), @@ -3297,7 +3598,7 @@ mod tests { let auth_file = write_proxy_auth_file("start-fail"); let (socket_path, request_log, handle) = spawn_podman_stub( "create-start-fail", - create_setup_responses(true) + create_setup_responses(true, sandbox_id) .into_iter() .chain(create_launch_responses().into_iter().take(7)) .chain([ diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 93a3f92966..d6ad260d0e 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -18,6 +18,13 @@ use openshell_driver_podman::{ComputeDriverService, PodmanComputeConfig, PodmanC #[command(name = "openshell-driver-podman")] #[command(version = VERSION)] struct Args { + /// Operator-owned JSON policy; omitted means driver config disabled and labels required. + #[arg( + long, + env = "OPENSHELL_DRIVER_ADMISSION_CONFIG_JSON", + default_value = "{}" + )] + admission_config_json: openshell_core::resource_admission::DriverAdmissionConfig, /// Public compute-driver Unix socket used by an external gateway. #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] bind_socket: Option, @@ -200,6 +207,8 @@ async fn main() -> Result<()> { ); let driver = PodmanComputeDriver::new(PodmanComputeConfig { + allow_driver_config: args.admission_config_json.allow_driver_config, + resource_admission: args.admission_config_json.resource_admission.clone(), socket_path: args.podman_socket, default_image: args.sandbox_image.unwrap_or_default(), image_pull_policy: args.sandbox_image_pull_policy, diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index f38fe901da..3f483577a8 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -8,6 +8,13 @@ The driver embeds libkrun, libkrunfw, the guest OCI unpacker, the portable guest ## How it fits together +Caller driver config is disabled by default, including upload workflows +encoded in driver JSON. GPU devices and their trusted VFIO plumbing are +temporarily exempt from approval labels; existing GPU selection validation +still applies. Private rootfs staging does not authorize arbitrary host paths. +The gateway passes its common policy to its managed VM subprocess. +See [resource admission configuration](../../docs/reference/gateway-config.mdx#external-resource-admission). + ```mermaid flowchart LR subgraph host["Host"] diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 68f1de47c8..92838af148 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -246,6 +246,12 @@ enum GuestImagePayloadSource { #[derive(Clone, serde::Serialize, serde::Deserialize)] #[serde(deny_unknown_fields)] pub struct VmDriverConfig { + /// Permit caller-supplied driver JSON. Does not waive resource admission. + #[serde(default)] + pub allow_driver_config: bool, + /// Operator-owned external attachment approval policy. + #[serde(default)] + pub resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig, pub grpc_endpoint: String, pub state_dir: PathBuf, pub launcher_bin: Option, @@ -363,6 +369,9 @@ impl Default for VmDriverConfig { fn default() -> Self { Self { grpc_endpoint: String::new(), + allow_driver_config: false, + resource_admission: + openshell_core::resource_admission::ResourceAdmissionConfig::default(), state_dir: PathBuf::from("target/openshell-vm-driver"), launcher_bin: None, default_image: String::new(), @@ -671,6 +680,7 @@ impl VmDriver { mut config: VmDriverConfig, lifecycle_extensions: LifecycleExtensionRegistry, ) -> Result { + config.resource_admission.validate()?; lifecycle_extensions .validate() .map_err(|err| err.message().to_string())?; @@ -1010,6 +1020,11 @@ impl VmDriver { #[must_use] pub fn capabilities(&self) -> GetCapabilitiesResponse { GetCapabilitiesResponse { + resource_admission_policy: openshell_core::resource_admission::DriverAdmissionConfig { + allow_driver_config: self.config.allow_driver_config, + resource_admission: self.config.resource_admission.clone(), + } + .acknowledgement(), driver_name: DRIVER_NAME.to_string(), driver_version: openshell_core::VERSION.to_string(), default_image: self.config.default_image.clone(), @@ -1047,6 +1062,10 @@ impl VmDriver { // gRPC API surface; boxing here would diverge from every other handler. #[allow(clippy::result_large_err)] pub fn validate_sandbox(&self, sandbox: &Sandbox) -> Result<(), Status> { + openshell_core::resource_admission::check_sandbox_driver_config( + self.config.allow_driver_config, + sandbox, + )?; validate_vm_sandbox(sandbox, self.config.gpu_enabled)?; let has_rootfs_tar = VmSandboxDriverConfig::from_sandbox(sandbox).is_ok_and(|c| c.rootfs_tar_path.is_some()); @@ -1062,6 +1081,7 @@ impl VmDriver { // gRPC API surface; boxing here would diverge from every other handler. #[allow(clippy::result_large_err)] pub async fn create_sandbox(&self, sandbox: &Sandbox) -> Result { + self.validate_sandbox(sandbox)?; info!( sandbox_id = %sandbox.id, sandbox_name = %sandbox.name, @@ -1832,6 +1852,12 @@ impl VmDriver { record.process.is_some() || record.provisioning_task.is_some(), ) }; + let mut sandbox = read_sandbox_request(&state_dir.join(SANDBOX_REQUEST_FILE)) + .await + .map_err(|error| { + Status::failed_precondition(format!("read VM admission provenance: {error}")) + })?; + self.validate_sandbox(&sandbox)?; if already_running { let active_generation = tokio::fs::read_to_string(state_dir.join(HOST_BOUNDARY_GENERATION_FILE)) @@ -1870,11 +1896,6 @@ impl VmDriver { ) .await .map_err(|error| Status::internal(format!("persist VM start generation: {error}")))?; - let mut sandbox = read_sandbox_request(&state_dir.join(SANDBOX_REQUEST_FILE)) - .await - .map_err(|err| { - Status::internal(format!("read sandbox start metadata failed: {err}")) - })?; let authentication = serde_json::from_slice::< openshell_core::jwt::SandboxLaunchAuthentication, >(&launch_authentication) @@ -2177,6 +2198,10 @@ impl VmDriver { clear_stop_marker: bool, reconciliation_span: &tracing::Span, ) -> bool { + if let Err(error) = self.validate_sandbox(&sandbox) { + warn!(sandbox_id = %sandbox.id, reason = %error.message(), "VM recovery denied by admission"); + return false; + } let has_rootfs_tar = VmSandboxDriverConfig::from_sandbox(&sandbox) .is_ok_and(|c| c.rootfs_tar_path.is_some()); @@ -8581,11 +8606,12 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let mut driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); driver.config.state_dir = temp.path().to_path_buf(); + let (old_authentication, old_session) = test_launch_authentication("old"); let sandbox = Sandbox { id: "sandbox-stopped".to_string(), name: "stopped".to_string(), spec: Some(SandboxSpec { - launch_authentication: test_launch_authentication("old").0, + launch_authentication: old_authentication, ..Default::default() }), ..Default::default() @@ -8609,7 +8635,7 @@ mod tests { }, ); - let (fresh_authentication, fresh_session) = test_launch_authentication("fresh"); + let (fresh_authentication, _) = test_launch_authentication("fresh"); let err = driver .start_sandbox( &sandbox.id, @@ -8620,7 +8646,7 @@ mod tests { .await .expect_err("start without an image should fail"); - assert_eq!(err.code(), Code::Internal); + assert_eq!(err.code(), Code::FailedPrecondition); assert!( tokio::fs::metadata(state_dir.join(SANDBOX_STOPPED_FILE)) .await @@ -8650,10 +8676,7 @@ mod tests { .launch_authentication, ) .expect("persisted launch authentication"); - assert_eq!( - persisted_authentication.supervisor.session_id, - fresh_session - ); + assert_eq!(persisted_authentication.supervisor.session_id, old_session); } fn test_launch_authentication(label: &str) -> (Vec, openshell_core::SandboxSessionId) { diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 65f37eae56..eedb86fecb 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -25,6 +25,13 @@ use tracing::info; #[command(version = VERSION)] #[allow(clippy::struct_excessive_bools)] struct Args { + /// Operator-owned JSON policy; omitted means driver config disabled and labels required. + #[arg( + long, + env = "OPENSHELL_DRIVER_ADMISSION_CONFIG_JSON", + default_value = "{}" + )] + admission_config_json: openshell_core::resource_admission::DriverAdmissionConfig, #[arg(long, hide = true, default_value_t = false)] internal_run_vm: bool, @@ -274,6 +281,8 @@ async fn main() -> Result<()> { } let driver = VmDriver::new(VmDriverConfig { + allow_driver_config: args.admission_config_json.allow_driver_config, + resource_admission: args.admission_config_json.resource_admission.clone(), grpc_endpoint: args .grpc_endpoint .ok_or_else(|| miette::miette!("OPENSHELL_GRPC_ENDPOINT is required"))?, diff --git a/crates/openshell-gateway/Cargo.toml b/crates/openshell-gateway/Cargo.toml index 32dcaeb24c..31e3975f0b 100644 --- a/crates/openshell-gateway/Cargo.toml +++ b/crates/openshell-gateway/Cargo.toml @@ -15,6 +15,7 @@ name = "openshell-gateway" path = "src/main.rs" [dependencies] +serde_json = { workspace = true } openshell-core = { path = "../openshell-core", default-features = false } openshell-server = { path = "../openshell-server", default-features = false } openshell-otel = { path = "../openshell-otel", optional = true } diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs index 86b7e37587..3b7bd6bf31 100644 --- a/crates/openshell-gateway/src/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -63,6 +63,8 @@ const COMPUTE_DRIVER_SOCKET_NAME: &str = "compute-driver.sock"; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] #[serde(default, deny_unknown_fields)] pub struct VmComputeConfig { + pub allow_driver_config: bool, + pub resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig, /// Working directory for VM driver sandbox state. pub state_dir: PathBuf, @@ -165,6 +167,7 @@ impl VmComputeConfig { /// Validate startup configuration without resolving binaries, creating /// state directories, spawning a process, or connecting a socket. pub fn validate_configuration(&self) -> Result<()> { + self.resource_admission.validate().map_err(Error::config)?; if self.grpc_endpoint.trim().is_empty() { return Err(Error::config( "grpc_endpoint is required when using the vm compute driver", @@ -232,6 +235,9 @@ impl Default for VmComputeConfig { fn default() -> Self { Self { state_dir: Self::default_state_dir(), + allow_driver_config: false, + resource_admission: + openshell_core::resource_admission::ResourceAdmissionConfig::default(), driver_dir: None, default_image: openshell_core::image::default_sandbox_image(), grpc_endpoint: String::new(), @@ -554,6 +560,13 @@ pub async fn spawn( command.stdout(Stdio::inherit()); command.stderr(Stdio::inherit()); command.arg("--bind-socket").arg(&socket_path); + command.arg("--admission-config-json").arg( + serde_json::to_string(&openshell_core::resource_admission::DriverAdmissionConfig { + allow_driver_config: vm_config.allow_driver_config, + resource_admission: vm_config.resource_admission.clone(), + }) + .map_err(|error| Error::config(error.to_string()))?, + ); command .arg("--expected-peer-pid") .arg(std::process::id().to_string()); diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index f264f4752f..6a2dfe44c2 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -114,6 +114,30 @@ pub struct DriverStartupContext<'a> { pub endpoint_overrides: &'a BTreeMap, } +/// Decode common controls without interpreting backend-specific driver fields. +pub fn admission_config_from_context( + context: DriverStartupContext<'_>, + name: &str, +) -> Result { + let mut table = toml::map::Map::new(); + if let Some(config) = context + .file + .and_then(|file| file.openshell.drivers.get(name)) + { + for field in ["allow_driver_config", "resource_admission"] { + if let Some(value) = config.get(field) { + table.insert(field.into(), value.clone()); + } + } + } + let policy: openshell_core::resource_admission::DriverAdmissionConfig = + toml::Value::Table(table).try_into().map_err(|error| { + Error::config(format!("invalid driver admission configuration: {error}")) + })?; + policy.validate().map_err(Error::config)?; + Ok(policy) +} + pub fn remote_driver_config_from_context( context: DriverStartupContext<'_>, name: &str, @@ -235,6 +259,37 @@ mod tests { } } + #[test] + fn common_admission_defaults_and_replacement_apply_to_every_driver() { + for name in ["kubernetes", "docker", "podman", "vm", "mxc", "external"] { + let defaults = admission_config_from_context(test_context(None), name).unwrap(); + assert!(!defaults.allow_driver_config); + assert!(defaults.resource_admission.enabled); + assert_eq!(defaults.resource_admission.required_labels.len(), 2); + let file: config_file::ConfigFile = toml::from_str(&format!( + "[openshell.drivers.{name}]\nallow_driver_config = true\n[openshell.drivers.{name}.resource_admission.required_labels]\n\"example.com/approved\" = \"yes\"\n" + )).unwrap(); + let custom = admission_config_from_context(test_context(Some(&file)), name).unwrap(); + assert!(custom.allow_driver_config); + assert_eq!( + custom.resource_admission.required_labels, + BTreeMap::from([("example.com/approved".into(), "yes".into())]) + ); + } + } + + #[test] + fn common_admission_rejects_empty_map_and_preserves_explicit_opt_out() { + let file: config_file::ConfigFile = + toml::from_str("[openshell.drivers.docker.resource_admission.required_labels]\n") + .unwrap(); + assert!(admission_config_from_context(test_context(Some(&file)), "docker").is_err()); + let file: config_file::ConfigFile = toml::from_str("[openshell.drivers.docker.resource_admission]\nenabled = false\nrequired_labels = {}\n").unwrap(); + let policy = admission_config_from_context(test_context(Some(&file)), "docker").unwrap(); + assert!(!policy.resource_admission.enabled); + assert!(!policy.allow_driver_config); + } + #[test] fn gateway_guest_tls_resolves_explicit_complete_bundle() { let dir = tempfile::tempdir().expect("temp dir"); diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 9f32743137..580d2f6dc6 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -455,6 +455,7 @@ impl AcquiredRemoteDriverEndpoint { #[derive(Debug, Clone)] struct RemoteComputeDriver { client: RemoteComputeDriverClient, + admission_acknowledgement: Arc>>, } type RemoteComputeDriverClient = ComputeDriverClient< @@ -465,12 +466,36 @@ impl RemoteComputeDriver { fn new(channel: Channel) -> Self { Self { client: ComputeDriverClient::with_interceptor(channel, TraceContextInterceptor), + admission_acknowledgement: Arc::new(StdMutex::new(None)), } } fn client(&self) -> RemoteComputeDriverClient { self.client.clone() } + + async fn verify_admission_policy(&self) -> Result<(), Status> { + let expected = self + .admission_acknowledgement + .lock() + .map_err(|_| Status::internal("admission handshake lock poisoned"))? + .clone() + .ok_or_else(|| Status::failed_precondition("driver admission handshake required"))?; + let current = self + .client() + .get_capabilities(GetCapabilitiesRequest { + gateway: Some(gateway_metadata(ExtensionFamily::Compute)), + }) + .await? + .into_inner() + .resource_admission_policy; + if current != expected { + return Err(Status::failed_precondition( + "remote driver admission policy changed; restart gateway after configuring matching policy", + )); + } + Ok(()) + } } #[tonic::async_trait] @@ -483,7 +508,12 @@ impl ComputeDriver for RemoteComputeDriver { ) -> Result, Status> { let mut client = self.client(); - client.get_capabilities(request).await + let response = client.get_capabilities(request).await?; + self.admission_acknowledgement + .lock() + .map_err(|_| Status::internal("admission handshake lock poisoned"))? + .get_or_insert_with(|| response.get_ref().resource_admission_policy.clone()); + Ok(response) } async fn authenticate_sandbox( @@ -505,6 +535,7 @@ impl ComputeDriver for RemoteComputeDriver { Status, > { let mut client = self.client(); + self.verify_admission_policy().await?; client.validate_sandbox_create(request).await } @@ -532,6 +563,7 @@ impl ComputeDriver for RemoteComputeDriver { ) -> Result, Status> { let mut client = self.client(); + self.verify_admission_policy().await?; client.create_sandbox(request).await } @@ -550,6 +582,7 @@ impl ComputeDriver for RemoteComputeDriver { ) -> Result, Status> { let mut client = self.client(); + self.verify_admission_policy().await?; client.start_sandbox(request).await } @@ -591,6 +624,8 @@ impl ComputeDriver for RemoteComputeDriver { #[derive(Clone)] pub struct ComputeRuntime { + admission: openshell_core::resource_admission::DriverAdmissionConfig, + admission_acknowledgement: String, driver: TracedDriver, driver_info: ComputeDriverInfoSnapshot, telemetry_compute_driver: TelemetryComputeDriver, @@ -688,6 +723,8 @@ impl ComputeRuntime { rootfs_tar_staging.sweep_orphans(); Ok(Self { driver: TracedDriver::new(driver, driver_name), + admission: openshell_core::resource_admission::DriverAdmissionConfig::default(), + admission_acknowledgement: capabilities.resource_admission_policy, driver_info, telemetry_compute_driver: TelemetryComputeDriver::custom(), driver_process, @@ -790,6 +827,33 @@ impl ComputeRuntime { &self.driver_info.name } + pub(crate) fn with_admission_policy( + mut self, + policy: openshell_core::resource_admission::DriverAdmissionConfig, + ) -> Result { + policy.verify_acknowledgement(&self.admission_acknowledgement)?; + if !policy.resource_admission.enabled { + warn!(driver = %self.driver_info.name, "External resource label admission is DISABLED"); + } + self.admission = policy; + Ok(self) + } + + pub(crate) fn validate_caller_driver_config( + &self, + template: Option<&SandboxTemplate>, + ) -> Result<(), Status> { + let selected = template + .map(|template| select_driver_config(&template.driver_config, &self.driver_info.name)) + .transpose() + .map_err(|error| *error)? + .flatten(); + openshell_core::resource_admission::check_driver_config( + self.admission.allow_driver_config, + selected.as_ref(), + ) + } + #[must_use] pub fn supports_sandbox_authentication(&self) -> bool { self.driver_info.supports_sandbox_authentication @@ -874,6 +938,12 @@ impl ComputeRuntime { } pub async fn validate_sandbox_create(&self, sandbox: &Sandbox) -> Result<(), Status> { + self.validate_caller_driver_config( + sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()), + )?; let mut driver_sandbox = driver_sandbox_from_public(sandbox, &self.driver_info.name) .map_err(|status| *status)?; // Peek, never consume: create runs the same path immediately after and @@ -947,6 +1017,12 @@ impl ComputeRuntime { lifecycle_guard: SandboxLifecycleGuard, global_guard: SandboxSyncGuard, ) -> Result { + self.validate_caller_driver_config( + sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()), + )?; let sandbox_id = sandbox.object_id().to_string(); let mut sandbox = sandbox; @@ -1445,6 +1521,13 @@ impl ComputeRuntime { )); } + self.validate_caller_driver_config( + current + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()), + )?; + let mut attempts = 0; let (previous, starting, launch_authentication) = loop { let phase = SandboxPhase::try_from(current.phase()).unwrap_or(SandboxPhase::Unknown); @@ -1474,7 +1557,6 @@ impl ComputeRuntime { "sandbox must be Stopped, Completed, or a failed main-process Error to start (current phase: {phase:?})" ))); } - if phase == SandboxPhase::Completed || is_failed_main_process_result(¤t) { self.cleanup_stopped_sandbox_sessions(¤t) .await @@ -1621,6 +1703,12 @@ impl ComputeRuntime { lifecycle_guard: SandboxLifecycleGuard, launch_authentication: Vec, ) -> Result { + self.validate_caller_driver_config( + starting + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()), + )?; let generation_id = sandbox_runtime_generation(&starting) .map_err(Status::failed_precondition)? .into_string(); @@ -2896,6 +2984,19 @@ impl ComputeRuntime { continue; } + if let Err(error) = self.validate_caller_driver_config( + sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()), + ) { + self.mark_sandbox_error(&sandbox, "ResourceAdmissionDenied", error.message()) + .await; + authentication_failed(sandbox.object_id()); + failed += 1; + continue; + } + let sandbox_name = sandbox.object_name().to_string(); let generation_id = match sandbox_runtime_generation(&sandbox) { Ok(generation) => generation.into_string(), @@ -3160,6 +3261,20 @@ impl ComputeRuntime { } } SandboxPhase::Starting => { + if let Err(error) = self.validate_caller_driver_config( + sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()), + ) { + self.mark_sandbox_error( + &sandbox, + "ResourceAdmissionDenied", + error.message(), + ) + .await; + continue; + } let sandbox_id = sandbox.object_id().to_string(); let sandbox_name = sandbox.object_name().to_string(); let driver_sandbox_id = sandbox_id.clone(); @@ -5883,6 +5998,9 @@ impl ComputeDriver for NoopTestDriver { Ok(tonic::Response::new( openshell_core::proto::compute::v1::GetCapabilitiesResponse { driver_name: "noop-test-driver".to_string(), + resource_admission_policy: + openshell_core::resource_admission::DriverAdmissionConfig::default() + .acknowledgement(), driver_version: "test".to_string(), default_image: "openshell/sandbox:test".to_string(), gateway_manages_lifecycle: false, @@ -6058,6 +6176,11 @@ pub fn new_test_runtime_with_driver( let supports_sandbox_authentication = driver.sandbox_authentication.is_some(); ComputeRuntime { driver: TracedDriver::new(driver, "test".to_string()), + admission: openshell_core::resource_admission::DriverAdmissionConfig { + allow_driver_config: true, + ..Default::default() + }, + admission_acknowledgement: String::new(), driver_info: ComputeDriverInfoSnapshot { name: driver_name.to_string(), driver_name: driver_name.to_string(), @@ -6474,6 +6597,9 @@ mod tests { ) -> Result, Status> { Ok(tonic::Response::new(GetCapabilitiesResponse { driver_name: "test-driver".to_string(), + resource_admission_policy: + openshell_core::resource_admission::DriverAdmissionConfig::default() + .acknowledgement(), driver_version: "test".to_string(), default_image: "openshell/sandbox:test".to_string(), gateway_manages_lifecycle: false, @@ -6858,6 +6984,9 @@ mod tests { ) -> Result, Status> { Ok(tonic::Response::new(GetCapabilitiesResponse { driver_name: "controlled-test-driver".to_string(), + resource_admission_policy: + openshell_core::resource_admission::DriverAdmissionConfig::default() + .acknowledgement(), driver_version: "test".to_string(), default_image: "openshell/sandbox:test".to_string(), gateway_manages_lifecycle: false, @@ -7094,6 +7223,11 @@ mod tests { let store = Arc::new(Store::connect("sqlite::memory:").await.unwrap()); ComputeRuntime { driver: TracedDriver::new(driver, "test-driver".to_string()), + admission: openshell_core::resource_admission::DriverAdmissionConfig { + allow_driver_config: true, + ..Default::default() + }, + admission_acknowledgement: String::new(), driver_info: ComputeDriverInfoSnapshot { name: driver_name.to_string(), driver_name: driver_name.to_string(), @@ -13275,7 +13409,7 @@ mod tests { let traceparents = driver.traceparents(); assert_eq!( traceparents.len(), - 8, + 10, "the client interceptor should cover every RPC" ); assert!( @@ -13346,7 +13480,7 @@ mod tests { .await .unwrap(); let store = Arc::new(Store::connect("sqlite::memory:").await.unwrap()); - let runtime = ComputeRuntime::new_remote_driver( + let mut runtime = ComputeRuntime::new_remote_driver( endpoint, store, SandboxIndex::new(), @@ -13356,6 +13490,7 @@ mod tests { ) .await .unwrap(); + runtime.admission.allow_driver_config = true; let mut sandbox = sandbox_record("sb-uds", "uds-sandbox", SandboxPhase::Provisioning); sandbox.spec = Some(SandboxSpec { log_level: "debug".to_string(), @@ -13387,8 +13522,10 @@ mod tests { runtime.validate_sandbox_create(&sandbox).await.unwrap(); runtime.create_sandbox(sandbox, None, false).await.unwrap(); let calls = driver.calls(); - assert_eq!(calls.len(), 3, "unexpected calls: {calls:?}"); - let validated = match &calls[1] { + assert_eq!(calls.len(), 5, "unexpected calls: {calls:?}"); + assert!(matches!(calls[1], FakeComputeDriverCall::GetCapabilities)); + assert!(matches!(calls[3], FakeComputeDriverCall::GetCapabilities)); + let validated = match &calls[2] { FakeComputeDriverCall::ValidateSandboxCreate { sandbox: Some(sandbox), } => sandbox, @@ -13411,7 +13548,7 @@ mod tests { Some(42) ); assert!(matches!( - &calls[2], + &calls[4], FakeComputeDriverCall::CreateSandbox { sandbox: Some(sandbox) } if sandbox.spec.as_ref().and_then(|spec| spec.policy.as_ref()) .is_some_and(|policy| policy.version == 42) @@ -13432,7 +13569,7 @@ mod tests { runtime.start_persisted_sandboxes().await.unwrap(); assert!(matches!( driver.calls().as_slice(), - [FakeComputeDriverCall::StartSandbox { sandbox_id, sandbox_name }] + [FakeComputeDriverCall::GetCapabilities, FakeComputeDriverCall::StartSandbox { sandbox_id, sandbox_name }] if sandbox_id == "sb-uds" && sandbox_name == "uds-sandbox" )); driver.clear_calls(); @@ -13458,6 +13595,54 @@ mod tests { } } + #[tokio::test] + #[cfg(unix)] + async fn remote_compute_driver_rejects_policy_changes_before_forwarding() { + use crate::test_support::{FakeComputeDriver, FakeComputeDriverCall}; + let dir = tempfile::tempdir().unwrap(); + let socket_path = dir.path().join("admission.sock"); + let driver = FakeComputeDriver::new(); + let _server = driver.serve_uds(&socket_path).unwrap(); + let endpoint = connect_remote_compute_driver("external-test", &socket_path) + .await + .unwrap(); + let remote = RemoteComputeDriver::new(endpoint.channel); + remote + .get_capabilities(Request::new(GetCapabilitiesRequest { + gateway: Some(gateway_metadata(ExtensionFamily::Compute)), + })) + .await + .unwrap(); + driver.set_admission_acknowledgement(String::new()); + driver.clear_calls(); + let sandbox = DriverSandbox::default(); + let status = remote + .validate_sandbox_create(Request::new(ValidateSandboxCreateRequest { + sandbox: Some(sandbox.clone()), + })) + .await + .unwrap_err(); + assert_eq!(status.code(), Code::FailedPrecondition); + assert!( + remote + .create_sandbox(Request::new(CreateSandboxRequest { + sandbox: Some(sandbox) + })) + .await + .is_err() + ); + assert!( + remote + .start_sandbox(Request::new(StartSandboxRequest::default())) + .await + .is_err() + ); + assert_eq!( + driver.calls(), + vec![FakeComputeDriverCall::GetCapabilities; 3] + ); + } + #[tokio::test] async fn create_sandbox_returns_resource_version_one() { let runtime = test_runtime(Arc::new(TestDriver::default())).await; diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index b75f464f0b..245b2f0b02 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -948,6 +948,10 @@ pub(super) async fn handle_create_sandbox_template( deletion_time: None, }); validate_sandbox_workload_template(&resolved)?; + let spec = sandbox_spec_from_user_workload_template(&resolved)?; + state + .compute + .validate_caller_driver_config(spec.template.as_ref())?; let labels_map = resolved.object_labels(); let labels_json = if labels_map.as_ref().is_none_or(HashMap::is_empty) { diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index ee3ef1768e..b41518761e 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -1663,6 +1663,8 @@ async fn build_compute_runtime( false, )?; let telemetry_compute_driver = driver.telemetry_compute_driver(registry); + let admission = + compute::driver_config::admission_config_from_context(driver_startup, driver.name())?; info!(driver = %driver.name(), "Using compute driver"); if config .gateway_jwt @@ -1744,6 +1746,9 @@ async fn build_compute_runtime( } }; + let runtime = runtime + .with_admission_policy(admission) + .map_err(Error::config)?; Ok(runtime.with_telemetry_compute_driver(telemetry_compute_driver)) } diff --git a/crates/openshell-server/src/test_support.rs b/crates/openshell-server/src/test_support.rs index a6b8d99e0b..dcbc2e1cf9 100644 --- a/crates/openshell-server/src/test_support.rs +++ b/crates/openshell-server/src/test_support.rs @@ -145,6 +145,9 @@ impl FakeComputeDriver { Self { state: Arc::new(Mutex::new(FakeComputeDriverState { capabilities: GetCapabilitiesResponse { + resource_admission_policy: + openshell_core::resource_admission::DriverAdmissionConfig::default() + .acknowledgement(), driver_name: "fake-compute-driver".to_string(), driver_version: "test".to_string(), default_image: "openshell/sandbox:test".to_string(), @@ -206,6 +209,10 @@ impl FakeComputeDriver { self.with_state(|state| state.calls.clear()); } + pub fn set_admission_acknowledgement(&self, acknowledgement: String) { + self.with_state(|state| state.capabilities.resource_admission_policy = acknowledgement); + } + #[cfg(unix)] pub fn serve_uds( &self, diff --git a/deploy/helm/openshell-workspace/templates/role.yaml b/deploy/helm/openshell-workspace/templates/role.yaml index 45b3beb831..3df8928496 100644 --- a/deploy/helm/openshell-workspace/templates/role.yaml +++ b/deploy/helm/openshell-workspace/templates/role.yaml @@ -10,6 +10,10 @@ metadata: labels: {{- include "openshell-workspace.labels" . | nindent 4 }} rules: + # Metadata-only application admission still requires Kubernetes get permission. + - apiGroups: [""] + resources: ["persistentvolumeclaims", "configmaps", "secrets"] + verbs: ["get"] - apiGroups: - agents.x-k8s.io resources: diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 160f543afa..9a900d0690 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -284,8 +284,11 @@ discovery endpoint or its TLS CA. | server.dbUrl | string | `"sqlite:/var/openshell/openshell.db"` | Gateway database URL (used for the default SQLite backend). | | server.defaultRuntimeClassName | string | `""` | Default Kubernetes runtimeClassName for sandbox pods. Applied when a CreateSandbox request does not specify one. Empty (default) = omit the field, using the cluster's default RuntimeClass. Set to a RuntimeClass name (e.g. "kata-containers", "nvidia") to apply it to all sandboxes that don't explicitly override it. | | server.disableTls | bool | `false` | Disable TLS entirely - the server listens on plaintext HTTP. Set to true when a reverse proxy / tunnel terminates TLS at the edge. | +| server.drivers.kubernetes.allowDriverConfig | bool | `false` | Allow caller driver JSON; external resources still require approval. | | server.drivers.kubernetes.operatorNamespaceFile | string | `""` | Path to a JSON file containing an array of namespace names allowed in operator mode. Hot-reloaded on change. | | server.drivers.kubernetes.operatorNamespaceLabel | string | `""` | K8s label selector for namespace discovery in operator mode. The driver watches namespaces matching this label. | +| server.drivers.kubernetes.resourceAdmission.enabled | bool | `true` | Require operator approval labels on external sandbox attachments (GPU attachments exempt). | +| server.drivers.kubernetes.resourceAdmission.requiredLabels | string | `nil` | Replacement label map; null uses built-in attachable/workspace labels. Empty map is invalid when enabled. | | server.drivers.kubernetes.workspaceMode | string | `"shared"` | How workspaces map to Kubernetes namespaces. "shared" (default): all sandboxes in a single namespace. "managed": auto-creates per-workspace namespaces. "operator": uses pre-provisioned namespaces. | | server.enableLoopbackServiceHttp | bool | `true` | Enable plaintext HTTP routing for loopback sandbox service URLs on TLS-enabled gateways. | | server.enableUserNamespaces | bool | `false` | Enable Kubernetes user namespace isolation (hostUsers: false) for sandbox pods. Requires Kubernetes 1.33+ with user namespace support available (beta through 1.35, GA in 1.36+), plus a supporting container runtime and Linux 5.12+. When enabled, container UID 0 maps to an unprivileged host UID and capabilities become namespaced. | diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 9265e0a96a..1df5c52467 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -9,6 +9,17 @@ metadata: labels: {{- include "openshell.labels" . | nindent 4 }} rules: + - apiGroups: ["node.k8s.io"] + resources: ["runtimeclasses"] + verbs: ["get"] + - apiGroups: ["scheduling.k8s.io"] + resources: ["priorityclasses"] + verbs: ["get"] + {{- if ne $workspaceMode "shared" }} + - apiGroups: [""] + resources: ["persistentvolumeclaims", "configmaps", "secrets"] + verbs: ["get"] + {{- end }} # Validate projected ServiceAccount tokens during sandbox bootstrap and # internal gateway peer authentication. - apiGroups: diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 0652db4b07..484f4109e4 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -138,6 +138,7 @@ data: {{- end }} [openshell.drivers.kubernetes] + allow_driver_config = {{ .Values.server.drivers.kubernetes.allowDriverConfig }} namespace = {{ include "openshell.sandboxNamespace" . | quote }} default_image = {{ .Values.server.sandboxImage | quote }} {{- if include "openshell.sandboxRuntimeImageOverrideEnabled" . }} @@ -215,6 +216,15 @@ data: sandbox_runtime_image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.sandboxRuntime.image.pullPolicy | quote }} {{- end }} + [openshell.drivers.kubernetes.resource_admission] + enabled = {{ .Values.server.drivers.kubernetes.resourceAdmission.enabled }} + {{- if ne .Values.server.drivers.kubernetes.resourceAdmission.requiredLabels nil }} + [openshell.drivers.kubernetes.resource_admission.required_labels] + {{- range $key, $value := .Values.server.drivers.kubernetes.resourceAdmission.requiredLabels }} + {{ $key | quote }} = {{ $value | quote }} + {{- end }} + {{- end }} + [openshell.drivers.kubernetes.managed_ssh_ingress] enabled = {{ .Values.networkPolicy.enabled }} gateway_namespace = {{ .Release.Namespace | quote }} diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index c781b46e3d..365ef3e9c7 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -10,6 +10,10 @@ metadata: labels: {{- include "openshell.labels" . | nindent 4 }} rules: + # Metadata-only application admission still requires Kubernetes get permission. + - apiGroups: [""] + resources: ["persistentvolumeclaims", "configmaps", "secrets"] + verbs: ["get"] - apiGroups: - agents.x-k8s.io resources: diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 4675cd9b6d..dbe8c0ae84 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -13,6 +13,45 @@ release: namespace: my-namespace tests: + - it: defaults to label admission with caller driver config disabled + template: templates/gateway-config.yaml + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^allow_driver_config\s*=\s*false$' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\.resource_admission\]\s*enabled\s*=\s*true' + - notMatchRegex: + path: data["gateway.toml"] + pattern: '\[openshell\.drivers\.kubernetes\.resource_admission\.required_labels\]' + + - it: preserves explicit admission opt-out without enabling driver config + template: templates/gateway-config.yaml + set: + server.drivers.kubernetes.resourceAdmission.enabled: false + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^allow_driver_config\s*=\s*false$' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\.resource_admission\]\s*enabled\s*=\s*false' + + - it: renders replacement labels and literal workspace substitution + template: templates/gateway-config.yaml + set: + server.drivers.kubernetes.allowDriverConfig: true + server.drivers.kubernetes.resourceAdmission.requiredLabels: + platform.example.com/team: '${workspace}' + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '"platform.example.com/team" = "\$\{workspace\}"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'openshell.ai/sandbox-attachable' + - it: identifies the gateway by chart fullname by default template: templates/gateway-config.yaml asserts: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index f395b7fa58..b90d59df04 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -261,6 +261,13 @@ server: # Kubernetes compute driver settings. drivers: kubernetes: + # -- Allow caller driver JSON; external resources still require approval. + allowDriverConfig: false + resourceAdmission: + # -- Require operator approval labels on external sandbox attachments (GPU attachments exempt). + enabled: true + # -- Replacement label map; null uses built-in attachable/workspace labels. Empty map is invalid when enabled. + requiredLabels: null # -- How workspaces map to Kubernetes namespaces. # "shared" (default): all sandboxes in a single namespace. # "managed": auto-creates per-workspace namespaces. diff --git a/docs/kubernetes/sandbox-runtime.mdx b/docs/kubernetes/sandbox-runtime.mdx index aa039ad027..63e6ccf9bf 100644 --- a/docs/kubernetes/sandbox-runtime.mdx +++ b/docs/kubernetes/sandbox-runtime.mdx @@ -87,6 +87,19 @@ cluster CNI enforces both ingress and egress policies for these namespaces. ## Bootstrap a Sandbox +Resource admission checks the requested external references before provisioning +and the actual workload Pod before releasing its scheduling gate. PVCs and +other supported external references require operator approval labels, including +same-namespace and read-only mounts. Unexpected references, changed resource +UIDs, and unsupported volume sources fail closed. GPU device attachments are +temporarily exempt. The driver rechecks persisted references on restart and +every 30 seconds during reconciliation; confirmed revocation suspends compute, +while transient lookup failures block new launches and recovery. + +See [External Resource Admission](../reference/gateway-config#external-resource-admission) +for label configuration and upgrade requirements. Admission cannot repair data +or credentials compromised before upgrade. + The driver creates each sandbox generation in a fail-closed order: 1. Create and validate the workload egress fence. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 4acd6808b1..b4489be115 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -531,6 +531,123 @@ The gateway refuses to start when a peer endpoint is configured on a multi-repli ## Driver References +### External Resource Admission + +Every compute driver disables caller-supplied `template.driver_config` by +default. An operator can enable it without disabling resource approval: + +```toml +[openshell.drivers.kubernetes] +allow_driver_config = true + +[openshell.drivers.kubernetes.resource_admission] +enabled = true + +[openshell.drivers.kubernetes.resource_admission.required_labels] +"openshell.ai/sandbox-attachable" = "true" +"openshell.ai/workspace" = "${workspace}" +``` + +Use the same fields under the selected Docker, Podman, VM, MXC, or external +driver table. Omitted admission settings use the labels above. A supplied map +replaces the defaults, and every entry must match. An explicitly empty map is +invalid while enforcement is enabled. `${workspace}` is the entire label value +and resolves to the authorized OpenShell workspace name, not a Kubernetes +namespace or caller label. Other substitutions and expressions are unsupported. +At least one fixed label is required so intentionally shared resources still +need explicit operator approval. + +The operator must label the referenced resource before use. Labels on a sandbox +do not approve its attachments. For example, a PVC used by workspace `team-a` +needs this metadata: + +```yaml +metadata: + labels: + openshell.ai/sandbox-attachable: "true" + openshell.ai/workspace: "team-a" +``` + +Kubernetes checks PVCs, RuntimeClasses, PriorityClasses, and credential/config +references selected through the OpenShell-owned Pod template. Docker and Podman +check engine volume labels. Read-only mounts still require approval. Namespaces +and operator-selected defaults do not waive checks. Newly provisioned private +storage and bootstrap material use driver ownership checks instead. + +RuntimeClasses, PriorityClasses, and image-pull Secrets are shared operator +infrastructure. They must match every fixed required label, but `${workspace}` +entries do not apply to them. PVCs and Secret or ConfigMap data exposed to a +sandbox must also match the resolved workspace label. Kubernetes API-server and +admission-webhook mutations of the live Pod are trusted cluster-operator behavior +and are not used as Workspace User authorization inputs. + +With Kubernetes admission enabled, configured image-pull Secrets in the gateway +namespace must carry the fixed approval labels. In managed mode OpenShell verifies +that source, creates the workspace namespace, and copies the Secret with gateway +ownership metadata. An existing target Secret is updated only when its ownership +metadata matches the gateway and workspace. Operator mode still requires the +approved Secret in each operator-managed namespace. Automatic service-account +tokens and projected token volumes are prohibited. + +GPU device attachments are temporarily exempt from label admission. Existing +GPU validation still applies. Explicit GPU device settings inside driver JSON +still require `allow_driver_config = true`; public GPU-count requests do not. +The exception does not approve unrelated host mounts or non-GPU devices. + +Raw host bind mounts and supplemental image mounts have no trusted label +resolver and are rejected while admission is enabled. Publisher image labels +cannot grant operator approval. Docker's existing unsupported image-mount +restriction remains. MXC host filesystem grants likewise require disabling +admission until a resolver exists. MXC's command config and VM upload workflows +encoded in driver JSON require explicit driver-config opt-in. + +`allow_driver_config` applies to nonempty caller config, including saved +templates and non-storage overrides. An absent config or `{}` does not require +opt-in. Drivers recheck admitted OpenShell input attachments before launch and +on restart; running Docker, Podman, and Kubernetes workloads are periodically +revalidated. +Confirmed revocation stops or suspends the workload. Metadata lookup outages +block new launches but do not alone stop existing workloads. Revalidation uses +the Docker observation loop (normally two seconds) or a 30-second Podman / +Kubernetes interval, plus lookup and retry time. These are not instantaneous +revocation guarantees. + +To explicitly opt out of label authorization: + +```toml +[openshell.drivers.kubernetes.resource_admission] +enabled = false +``` + +This does not enable driver JSON or bypass existing path, ownership, namespace, +or isolation validation. It relinquishes external-resource label protection +for that driver. Restart the gateway and driver after policy changes. Only +trusted operators should control approval labels, runtime APIs, and resource +replacement; application admission is not atomic mount authorization. + +Before upgrading, inventory and approve legitimate external resources. Legacy +sandbox state without verifiable admission provenance requires recreation; +OpenShell does not automatically approve old attachments. Remove stale labels +before reusing a deleted workspace name. A custom approval-only map permits +cross-workspace sharing and should be an intentional operator decision. + +Standalone Kubernetes, Podman, and VM drivers accept the common policy as +`--admission-config-json` or `OPENSHELL_DRIVER_ADMISSION_CONFIG_JSON`; `{}` uses +secure defaults. Docker also accepts the fields in its driver TOML file. The +gateway-managed VM process receives the selected gateway policy automatically. +Operator-managed external processes must be configured separately. Their +versioned `GetCapabilities.resource_admission_policy` acknowledgement must +match the gateway policy; legacy drivers require an explicit admission opt-out. +A changed acknowledgement blocks subsequent remote validation/create/start +calls until gateway and driver configuration agree again. + +Helm exposes `server.drivers.kubernetes.allowDriverConfig`, +`resourceAdmission.enabled`, and `resourceAdmission.requiredLabels`. The label +map defaults to `null`, which uses code defaults without merging those keys +into a custom map. Admission reads need Kubernetes `get` permissions even +though only metadata is requested. Never mark gateway database storage as +attachable. + Each example is a complete TOML file for one compute driver. The examples repeat `[openshell]` and `[openshell.gateway]` so they stay copyable, and the driver tables list the accepted driver-specific keys. Drivers receive only their own tables, and the gateway rejects unknown gateway and driver fields. Kubernetes configurations set `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`. Docker configurations use `sandbox_label`; the legacy `sandbox_namespace` key is rejected. diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 3291e48d4c..263cc429a4 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -10,6 +10,14 @@ position: 4 The gateway's configured compute driver determines how OpenShell creates each sandbox. The CLI workflow stays the same across drivers: you create, connect to, inspect, stop, start, and delete sandboxes through the gateway API. +Caller driver config is disabled by default across drivers. Enabling it does +not authorize external attachments: referenced resources must carry matching +operator-controlled approval labels. GPU device attachments are temporarily +exempt. Host bind mounts, supplemental image mounts, and MXC host filesystem +grants lack a trusted label resolver and are unavailable with admission enabled. +See [External Resource Admission](gateway-config#external-resource-admission) +for configuration, migration, and the explicit unsafe opt-out. + Most compute drivers run the OpenShell supervisor inside the sandbox workload. The supervisor launches the agent process, applies policy, routes egress through the proxy, injects configured credentials, and maintains the gateway session. @@ -199,7 +207,8 @@ For GPU-backed Docker sandboxes, configure Docker CDI before starting the gatewa Docker driver config accepts user-supplied `volume` and `tmpfs` mounts. It also accepts `bind` mounts when `[openshell.drivers.docker]` sets -`enable_bind_mounts = true` in `gateway.toml`. See Docker's [storage documentation](https://docs.docker.com/engine/storage/) for more information. +`enable_bind_mounts = true` and explicitly disables resource admission in +`gateway.toml`. All examples require `allow_driver_config = true`. See Docker's [storage documentation](https://docs.docker.com/engine/storage/) for more information. Docker local-driver named volumes created with bind options also expose gateway-host paths, so OpenShell treats them like bind mounts and requires `enable_bind_mounts = true`. @@ -207,7 +216,8 @@ gateway-host paths, so OpenShell treats them like bind mounts and requires Use a `volume` mount for existing Docker named volumes: ```shell -docker volume create openshell-work +docker volume create --label openshell.ai/sandbox-attachable=true \ + --label openshell.ai/workspace=default openshell-work openshell sandbox create \ --driver-config-json '{"docker":{"mounts":[{"type":"volume","source":"openshell-work","target":"/sandbox/work","read_only":false}]}}' \ @@ -221,11 +231,16 @@ workspace isolation and filesystem policy. Use them only when you understand and accept that loss of isolation. -Use a `bind` mount only after enabling it in the Docker driver table: +Raw paths cannot satisfy label admission. The following unsafe opt-out permits +bind mounts and removes label protection for all Docker attachments: ```toml [openshell.drivers.docker] +allow_driver_config = true enable_bind_mounts = true + +[openshell.drivers.docker.resource_admission] +enabled = false ``` ```shell @@ -281,9 +296,11 @@ your Podman machine uses a non-standard host-loopback address, or set ### Podman Driver Config Mounts -Podman driver config accepts user-supplied `volume`, `tmpfs`, and `image` -mounts. It also accepts `bind` mounts when `[openshell.drivers.podman]` sets -`enable_bind_mounts = true` in `gateway.toml`. Podman local-driver named +Podman driver config accepts user-supplied `volume` and `tmpfs` mounts. +All examples require `allow_driver_config = true`. Supplemental `image` mounts +require disabling resource admission. It also accepts `bind` mounts when +`[openshell.drivers.podman]` sets `enable_bind_mounts = true` and explicitly +disables resource admission in `gateway.toml`. Podman local-driver named volumes created with bind options also expose gateway-host paths, so OpenShell treats them like bind mounts and requires `enable_bind_mounts = true`. Host bind mounts expose gateway host paths to sandbox requests, so they are @@ -292,7 +309,8 @@ disabled by default. Use a `volume` mount for existing Podman named volumes: ```shell -podman volume create openshell-work +podman volume create --label openshell.ai/sandbox-attachable=true \ + --label openshell.ai/workspace=default openshell-work openshell sandbox create \ --driver-config-json '{"podman":{"mounts":[{"type":"volume","source":"openshell-work","target":"/sandbox/work","read_only":false}]}}' \ @@ -306,11 +324,16 @@ workspace isolation and filesystem policy. Use them only when you understand and accept that loss of isolation. -Use a `bind` mount only after enabling it in the Podman driver table: +Raw paths cannot satisfy label admission. The following unsafe opt-out permits +bind mounts and removes label protection for all Podman attachments: ```toml [openshell.drivers.podman] +allow_driver_config = true enable_bind_mounts = true + +[openshell.drivers.podman.resource_admission] +enabled = false ``` ```shell diff --git a/proto/compute_driver.proto b/proto/compute_driver.proto index d7875d67e6..f278174b5c 100644 --- a/proto/compute_driver.proto +++ b/proto/compute_driver.proto @@ -102,6 +102,10 @@ message GetCapabilitiesResponse { uint64 rootfs_tar_max_bytes = 11; // Compute extension protocol metadata. Required for protocol negotiation. openshell.extension.v1.PeerMetadata extension = 12; + // Versioned effective operator admission policy (v1: followed by JSON). + // Gateways require an exact policy match before activating the driver. + // Empty denotes a legacy driver and requires explicit admission opt-out. + string resource_admission_policy = 13; } message AuthenticateSandboxRequest { diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 67fb2288af..5fb77c8931 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -92,6 +92,22 @@ Use gateway metadata, deployment values, or the user's setup notes to identify t Before debugging the compute platform, inspect gateway logs for failures in dependencies initialized before the listener becomes ready. +For resource-admission failures, distinguish disabled caller driver config from +missing resource approval. Helm defaults `server.drivers.kubernetes.allowDriverConfig` +to false and `resourceAdmission.enabled` to true. Existing PVCs, Secrets, +ConfigMaps, RuntimeClasses, and PriorityClasses need matching administrator-owned +labels; namespace membership and read-only access do not grant approval. Inspect +metadata only when diagnosing credentials. GPU devices do not need labels. +RuntimeClasses, PriorityClasses, and image-pull Secrets match fixed approval +labels but do not use the workspace placeholder. In managed mode, inspect the +approved source image-pull Secret in the gateway namespace and the gateway-owned +copy in the workspace namespace. Legacy workloads without admission provenance need recreation. Do not +automatically label control-plane resources or disable enforcement as a repair. + +For out-of-tree compute drivers, also check that their versioned admission-policy +acknowledgement matches the gateway's policy. Configure standalone driver policy +through its administrator-owned `--admission-config-json` option. + For out-of-tree compute drivers, confirm the selected driver name and socket agree across CLI flags or `gateway.toml`, and that the operator-owned driver is running before the gateway starts: ```bash diff --git a/skills/openshell-cli/SKILL.md b/skills/openshell-cli/SKILL.md index b6711e5140..89c707d424 100644 --- a/skills/openshell-cli/SKILL.md +++ b/skills/openshell-cli/SKILL.md @@ -301,8 +301,15 @@ openshell sandbox template create gpu-kata \ openshell sandbox create --name my-sandbox --template gpu-kata --provider my-github -- claude ``` -Direct `sandbox create --driver-config-json` remains valid for one-off -creates. Put driver config on a template only when it should be reused. +Driver config is disabled by default. These template and one-off +`sandbox create --driver-config-json` examples require the administrator to set +`allow_driver_config = true` for the selected driver. This does not waive +resource admission: external attachments need administrator-controlled approval +labels on the actual resources, not sandbox labels. GPU device attachments +are temporarily exempt from labels; the public `--gpu` flag needs no driver +config opt-in. Consult the published gateway configuration reference before +changing admission settings; do not recommend disabling admission to bypass a +denial. Put driver config on a template only when it should be reused. ### Manage sandbox workload templates From 0b76f7aa69c438c5208c6c1a3cf3bc94fac32469 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Mon, 21 Sep 2026 20:11:40 -0700 Subject: [PATCH 2/8] fix(drivers): address resource admission review findings Signed-off-by: Drew Newberry --- crates/openshell-driver-docker/src/lib.rs | 14 ++++++++++---- crates/openshell-driver-docker/src/tests.rs | 15 +++++++++++++++ .../openshell/tests/gateway_config_test.yaml | 4 ++++ docs/kubernetes/sandbox-runtime.mdx | 19 +++++++++++-------- 4 files changed, 40 insertions(+), 12 deletions(-) diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index fd41581c67..bc2350f508 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -1649,10 +1649,7 @@ impl DockerComputeDriver { let _ = remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config) .await; cleanup_docker_boundary_state(sandbox, &self.config); - return Err(DockerProvisioningFailure::from_status( - "ResourceAdmissionDenied", - status, - )); + return Err(DockerProvisioningFailure::from_admission_status(status)); } }; create_body @@ -3449,6 +3446,15 @@ impl DockerProvisioningFailure { fn from_status(reason: &'static str, status: Status) -> Self { Self::new(reason, status.message()) } + + fn from_admission_status(status: Status) -> Self { + let reason = if status.code() == tonic::Code::FailedPrecondition { + "ResourceAdmissionDenied" + } else { + "ResourceAdmissionLookupFailed" + }; + Self::from_status(reason, status) + } } fn sandbox_image(sandbox: &DriverSandbox) -> Option { diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 0aa8f5c5c7..c6d7ff5f5e 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -3469,3 +3469,18 @@ async fn different_start_generation_is_rejected() { assert_eq!(error.code(), tonic::Code::FailedPrecondition); assert!(error.message().contains("generation-one")); } + +#[test] +fn admission_provisioning_failure_distinguishes_denials_from_lookup_failures() { + let denied = DockerProvisioningFailure::from_admission_status(Status::failed_precondition( + "volume is not admitted", + )); + assert_eq!(denied.reason, "ResourceAdmissionDenied"); + assert_eq!(denied.message, "volume is not admitted"); + + let lookup = DockerProvisioningFailure::from_admission_status(Status::internal( + "inspect docker volume failed", + )); + assert_eq!(lookup.reason, "ResourceAdmissionLookupFailed"); + assert_eq!(lookup.message, "inspect docker volume failed"); +} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index dbe8c0ae84..e4e31b6052 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -43,8 +43,12 @@ tests: set: server.drivers.kubernetes.allowDriverConfig: true server.drivers.kubernetes.resourceAdmission.requiredLabels: + platform.example.com/sandbox-attachable: "approved" platform.example.com/team: '${workspace}' asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '"platform.example.com/sandbox-attachable" = "approved"' - matchRegex: path: data["gateway.toml"] pattern: '"platform.example.com/team" = "\$\{workspace\}"' diff --git a/docs/kubernetes/sandbox-runtime.mdx b/docs/kubernetes/sandbox-runtime.mdx index 63e6ccf9bf..ffa7350f2e 100644 --- a/docs/kubernetes/sandbox-runtime.mdx +++ b/docs/kubernetes/sandbox-runtime.mdx @@ -87,14 +87,17 @@ cluster CNI enforces both ingress and egress policies for these namespaces. ## Bootstrap a Sandbox -Resource admission checks the requested external references before provisioning -and the actual workload Pod before releasing its scheduling gate. PVCs and -other supported external references require operator approval labels, including -same-namespace and read-only mounts. Unexpected references, changed resource -UIDs, and unsupported volume sources fail closed. GPU device attachments are -temporarily exempt. The driver rechecks persisted references on restart and -every 30 seconds during reconciliation; confirmed revocation suspends compute, -while transient lookup failures block new launches and recovery. +Resource admission checks external references selected through the OpenShell-owned +Pod template before provisioning and again before releasing the workload's +scheduling gate. Kubernetes API-server and admission-webhook mutations of the +live Pod are trusted cluster-operator behavior and are not used as Workspace User +authorization inputs. PVCs and other supported external references require +operator approval labels, including same-namespace and read-only mounts. +Unexpected references, changed resource UIDs, and unsupported volume sources fail +closed. GPU device attachments are temporarily exempt. The driver rechecks +persisted references on restart and every 30 seconds during reconciliation; +confirmed revocation suspends compute, while transient lookup failures block new +launches and recovery. See [External Resource Admission](../reference/gateway-config#external-resource-admission) for label configuration and upgrade requirements. Admission cannot repair data From 333e56b6bc6c3ee8fdb045d8ea41366f8f8dca83 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 22 Sep 2026 09:39:16 -0700 Subject: [PATCH 3/8] fix(core): reserve driver-owned admission labels Signed-off-by: Drew Newberry --- .../openshell-core/src/resource_admission.rs | 29 +++++++++++++++++++ docs/reference/gateway-config.mdx | 6 ++++ 2 files changed, 35 insertions(+) diff --git a/crates/openshell-core/src/resource_admission.rs b/crates/openshell-core/src/resource_admission.rs index 05c4b81318..514e0b0e42 100644 --- a/crates/openshell-core/src/resource_admission.rs +++ b/crates/openshell-core/src/resource_admission.rs @@ -8,7 +8,11 @@ use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; +use crate::driver_utils::{LABEL_GATEWAY_ID, LABEL_MANAGED_BY, LABEL_SANDBOX_WORKSPACE}; + const WORKSPACE_PLACEHOLDER: &str = "${workspace}"; +const RESERVED_LABEL_KEYS: [&str; 3] = + [LABEL_MANAGED_BY, LABEL_GATEWAY_ID, LABEL_SANDBOX_WORKSPACE]; /// Label policy shared by every compute driver. A supplied map replaces defaults. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -135,6 +139,11 @@ impl ResourceAdmissionConfig { ); } for (key, value) in &self.required_labels { + if RESERVED_LABEL_KEYS.contains(&key.as_str()) { + return Err(format!( + "resource admission label key is reserved for driver-owned metadata: {key}" + )); + } if !valid_label_key(key) { return Err(format!("invalid resource admission label key: {key}")); } @@ -317,6 +326,26 @@ mod tests { ); } + #[test] + fn rejects_driver_owned_label_keys() { + assert!(ResourceAdmissionConfig::default().validate().is_ok()); + for key in RESERVED_LABEL_KEYS { + let policy = ResourceAdmissionConfig { + required_labels: BTreeMap::from([ + ("example.com/approved".into(), "true".into()), + (key.into(), "operator-value".into()), + ]), + ..Default::default() + }; + assert_eq!( + policy.validate(), + Err(format!( + "resource admission label key is reserved for driver-owned metadata: {key}" + )) + ); + } + } + #[test] fn rejects_invalid_labels_and_interpolation() { for value in [ diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index b4489be115..0d7be5592c 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -557,6 +557,12 @@ namespace or caller label. Other substitutions and expressions are unsupported. At least one fixed label is required so intentionally shared resources still need explicit operator approval. +The following label keys are reserved for driver-owned metadata and cannot be +used in `required_labels`: `openshell.ai/managed-by`, `openshell.ai/gateway-id`, +and `openshell.ai/sandbox-workspace`. The default workspace admission key is +`openshell.ai/workspace`, which is distinct from the reserved sandbox metadata +key. + The operator must label the referenced resource before use. Labels on a sandbox do not approve its attachments. For example, a PVC used by workspace `team-a` needs this metadata: From 1c5b4ed5f2db1f804261d12c50203caae7a104a7 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 22 Sep 2026 09:47:52 -0700 Subject: [PATCH 4/8] fix(core): clarify workspace admission label Signed-off-by: Drew Newberry --- crates/openshell-core/src/resource_admission.rs | 9 ++++++--- crates/openshell-driver-docker/src/tests.rs | 5 ++++- .../src/resource_admission.rs | 4 ++-- crates/openshell-driver-podman/src/driver.rs | 4 ++-- deploy/helm/openshell/README.md | 2 +- deploy/helm/openshell/values.yaml | 2 +- docs/reference/gateway-config.mdx | 10 +++++----- docs/reference/sandbox-compute-drivers.mdx | 4 ++-- 8 files changed, 23 insertions(+), 17 deletions(-) diff --git a/crates/openshell-core/src/resource_admission.rs b/crates/openshell-core/src/resource_admission.rs index 514e0b0e42..7d20c405f4 100644 --- a/crates/openshell-core/src/resource_admission.rs +++ b/crates/openshell-core/src/resource_admission.rs @@ -29,7 +29,7 @@ impl Default for ResourceAdmissionConfig { required_labels: BTreeMap::from([ ("openshell.ai/sandbox-attachable".into(), "true".into()), ( - "openshell.ai/workspace".into(), + "openshell.ai/sandbox-attachable-workspace".into(), WORKSPACE_PLACEHOLDER.into(), ), ]), @@ -280,7 +280,10 @@ mod tests { fn labels(workspace: &str) -> BTreeMap { BTreeMap::from([ ("openshell.ai/sandbox-attachable".into(), "true".into()), - ("openshell.ai/workspace".into(), workspace.into()), + ( + "openshell.ai/sandbox-attachable-workspace".into(), + workspace.into(), + ), ]) } @@ -312,7 +315,7 @@ mod tests { serde_json::from_str(r#"{"required_labels":{}}"#).unwrap(); assert!(empty.validate().is_err()); let workspace_only: ResourceAdmissionConfig = serde_json::from_str( - r#"{"required_labels":{"openshell.ai/workspace":"${workspace}"}}"#, + r#"{"required_labels":{"openshell.ai/sandbox-attachable-workspace":"${workspace}"}}"#, ) .unwrap(); assert!(workspace_only.validate().is_err()); diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index c6d7ff5f5e..4313f776f3 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -318,7 +318,10 @@ async fn live_docker_resource_admission_checks_native_volume_labels() { let labels = workspace.map(|workspace| { HashMap::from([ ("openshell.ai/sandbox-attachable".into(), "true".into()), - ("openshell.ai/workspace".into(), workspace.into()), + ( + "openshell.ai/sandbox-attachable-workspace".into(), + workspace.into(), + ), ]) }); driver diff --git a/crates/openshell-driver-kubernetes/src/resource_admission.rs b/crates/openshell-driver-kubernetes/src/resource_admission.rs index 26e5e0a95d..5f0e369f5d 100644 --- a/crates/openshell-driver-kubernetes/src/resource_admission.rs +++ b/crates/openshell-driver-kubernetes/src/resource_admission.rs @@ -258,11 +258,11 @@ mod tests { for (labels, allowed) in [ (serde_json::json!({}), false), ( - serde_json::json!({"openshell.ai/sandbox-attachable":"true","openshell.ai/workspace":"other"}), + serde_json::json!({"openshell.ai/sandbox-attachable":"true","openshell.ai/sandbox-attachable-workspace":"other"}), false, ), ( - serde_json::json!({"openshell.ai/sandbox-attachable":"true","openshell.ai/workspace":"team-a"}), + serde_json::json!({"openshell.ai/sandbox-attachable":"true","openshell.ai/sandbox-attachable-workspace":"team-a"}), true, ), ] { diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index d366ebe00e..a8d852562d 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -2980,11 +2980,11 @@ mod tests { (serde_json::json!(null), false), (serde_json::json!({}), false), ( - serde_json::json!({"openshell.ai/sandbox-attachable":"true","openshell.ai/workspace":"other"}), + serde_json::json!({"openshell.ai/sandbox-attachable":"true","openshell.ai/sandbox-attachable-workspace":"other"}), false, ), ( - serde_json::json!({"openshell.ai/sandbox-attachable":"true","openshell.ai/workspace":"team-a"}), + serde_json::json!({"openshell.ai/sandbox-attachable":"true","openshell.ai/sandbox-attachable-workspace":"team-a"}), true, ), ] { diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 9a900d0690..fac1f85b9d 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -288,7 +288,7 @@ discovery endpoint or its TLS CA. | server.drivers.kubernetes.operatorNamespaceFile | string | `""` | Path to a JSON file containing an array of namespace names allowed in operator mode. Hot-reloaded on change. | | server.drivers.kubernetes.operatorNamespaceLabel | string | `""` | K8s label selector for namespace discovery in operator mode. The driver watches namespaces matching this label. | | server.drivers.kubernetes.resourceAdmission.enabled | bool | `true` | Require operator approval labels on external sandbox attachments (GPU attachments exempt). | -| server.drivers.kubernetes.resourceAdmission.requiredLabels | string | `nil` | Replacement label map; null uses built-in attachable/workspace labels. Empty map is invalid when enabled. | +| server.drivers.kubernetes.resourceAdmission.requiredLabels | string | `nil` | Replacement label map; null uses the built-in admission labels. Empty map is invalid when enabled. | | server.drivers.kubernetes.workspaceMode | string | `"shared"` | How workspaces map to Kubernetes namespaces. "shared" (default): all sandboxes in a single namespace. "managed": auto-creates per-workspace namespaces. "operator": uses pre-provisioned namespaces. | | server.enableLoopbackServiceHttp | bool | `true` | Enable plaintext HTTP routing for loopback sandbox service URLs on TLS-enabled gateways. | | server.enableUserNamespaces | bool | `false` | Enable Kubernetes user namespace isolation (hostUsers: false) for sandbox pods. Requires Kubernetes 1.33+ with user namespace support available (beta through 1.35, GA in 1.36+), plus a supporting container runtime and Linux 5.12+. When enabled, container UID 0 maps to an unprivileged host UID and capabilities become namespaced. | diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index b90d59df04..99ad6e9c0f 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -266,7 +266,7 @@ server: resourceAdmission: # -- Require operator approval labels on external sandbox attachments (GPU attachments exempt). enabled: true - # -- Replacement label map; null uses built-in attachable/workspace labels. Empty map is invalid when enabled. + # -- Replacement label map; null uses the built-in admission labels. Empty map is invalid when enabled. requiredLabels: null # -- How workspaces map to Kubernetes namespaces. # "shared" (default): all sandboxes in a single namespace. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 0d7be5592c..4967508a2e 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -545,7 +545,7 @@ enabled = true [openshell.drivers.kubernetes.resource_admission.required_labels] "openshell.ai/sandbox-attachable" = "true" -"openshell.ai/workspace" = "${workspace}" +"openshell.ai/sandbox-attachable-workspace" = "${workspace}" ``` Use the same fields under the selected Docker, Podman, VM, MXC, or external @@ -559,9 +559,9 @@ need explicit operator approval. The following label keys are reserved for driver-owned metadata and cannot be used in `required_labels`: `openshell.ai/managed-by`, `openshell.ai/gateway-id`, -and `openshell.ai/sandbox-workspace`. The default workspace admission key is -`openshell.ai/workspace`, which is distinct from the reserved sandbox metadata -key. +and `openshell.ai/sandbox-workspace`. The default admission key +`openshell.ai/sandbox-attachable-workspace` is distinct from the reserved +sandbox metadata key. The operator must label the referenced resource before use. Labels on a sandbox do not approve its attachments. For example, a PVC used by workspace `team-a` @@ -571,7 +571,7 @@ needs this metadata: metadata: labels: openshell.ai/sandbox-attachable: "true" - openshell.ai/workspace: "team-a" + openshell.ai/sandbox-attachable-workspace: "team-a" ``` Kubernetes checks PVCs, RuntimeClasses, PriorityClasses, and credential/config diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 263cc429a4..76e0c85355 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -217,7 +217,7 @@ Use a `volume` mount for existing Docker named volumes: ```shell docker volume create --label openshell.ai/sandbox-attachable=true \ - --label openshell.ai/workspace=default openshell-work + --label openshell.ai/sandbox-attachable-workspace=default openshell-work openshell sandbox create \ --driver-config-json '{"docker":{"mounts":[{"type":"volume","source":"openshell-work","target":"/sandbox/work","read_only":false}]}}' \ @@ -310,7 +310,7 @@ Use a `volume` mount for existing Podman named volumes: ```shell podman volume create --label openshell.ai/sandbox-attachable=true \ - --label openshell.ai/workspace=default openshell-work + --label openshell.ai/sandbox-attachable-workspace=default openshell-work openshell sandbox create \ --driver-config-json '{"podman":{"mounts":[{"type":"volume","source":"openshell-work","target":"/sandbox/work","read_only":false}]}}' \ From e9a21c33e0256f06f47ee0d8090e44644f1e1ee9 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 22 Sep 2026 10:24:53 -0700 Subject: [PATCH 5/8] fix(drivers): clarify resource admission failures Signed-off-by: Drew Newberry --- crates/openshell-driver-docker/src/lib.rs | 16 +- crates/openshell-driver-docker/src/tests.rs | 19 +- .../openshell-driver-kubernetes/src/driver.rs | 16 +- .../src/resource_admission.rs | 194 ++++++++++++++++-- crates/openshell-driver-podman/src/driver.rs | 27 ++- crates/openshell-driver-vm/src/driver.rs | 61 +++++- 6 files changed, 298 insertions(+), 35 deletions(-) diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index bc2350f508..c76c4c838b 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -1141,7 +1141,13 @@ impl DockerComputeDriver { identities.insert(source.clone(), docker_volume_identity(&volume)); self.config .resource_admission - .admit(workspace, &volume.labels)?; + .admit(workspace, &volume.labels) + .map_err(|error| { + Status::new( + error.code(), + format!("docker volume '{source}': {}", error.message()), + ) + })?; if !self.config.enable_bind_mounts && docker_volume_is_bind_backed(&volume) { return Err(Status::failed_precondition(format!( @@ -1254,7 +1260,13 @@ impl DockerComputeDriver { actual.insert(name.to_string(), docker_volume_identity(&volume)); self.config .resource_admission - .admit(workspace, &volume.labels)?; + .admit(workspace, &volume.labels) + .map_err(|error| { + Status::new( + error.code(), + format!("docker volume '{name}': {}", error.message()), + ) + })?; if !self.config.enable_bind_mounts && docker_volume_is_bind_backed(&volume) { return Err(Status::failed_precondition( diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 4313f776f3..aa97d0ff09 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -339,12 +339,19 @@ async fn live_docker_resource_admission_checks_native_volume_labels() { "type":"volume", "source":name, "target":"/external", "read_only":read_only }]})) .unwrap(); - results.push( - driver - .validate_user_volume_mounts_available(&mounts, "team-a") - .await - .is_ok(), - ); + let result = driver + .validate_user_volume_mounts_available(&mounts, "team-a") + .await; + if !approved { + assert!( + result + .as_ref() + .unwrap_err() + .message() + .contains(&format!("docker volume '{name}'")) + ); + } + results.push(result.is_ok()); } driver .docker diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 7b64434868..9388012fb4 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1270,7 +1270,17 @@ impl KubernetesComputeDriver { .into_iter() .flat_map(|labels| labels.iter()), ) - .map_err(admission_error)?; + .map_err(|error| { + admission_error(tonic::Status::new( + error.code(), + format!( + "Secret '{}/{}': {}", + self.config.namespace, + secret_name, + error.message() + ), + )) + })?; let existing = tokio::time::timeout(KUBE_API_TIMEOUT, target_api.get_opt(secret_name)) .await @@ -10889,6 +10899,10 @@ mod tests { .await .expect_err("unapproved source must not be copied"); assert!(matches!(error, KubernetesDriverError::Precondition(_))); + assert!( + error.to_string().contains("Secret 'openshell/regcred'"), + "unexpected error: {error}" + ); } #[test] diff --git a/crates/openshell-driver-kubernetes/src/resource_admission.rs b/crates/openshell-driver-kubernetes/src/resource_admission.rs index 5f0e369f5d..d19d9c9e0a 100644 --- a/crates/openshell-driver-kubernetes/src/resource_admission.rs +++ b/crates/openshell-driver-kubernetes/src/resource_admission.rs @@ -32,6 +32,48 @@ struct Reference { scope: Scope, } +fn resource_description(reference: &Reference, namespace: &str, cluster: bool) -> String { + if cluster { + format!("{} '{}'", reference.kind, reference.name) + } else { + format!("{} '{}/{}'", reference.kind, namespace, reference.name) + } +} + +fn contextualize(status: Status, resource: &str) -> Status { + Status::new(status.code(), format!("{resource}: {}", status.message())) +} + +fn metadata_lookup_error(error: kube::Error, resource: &str) -> Status { + match error { + kube::Error::Api(response) if response.code == 404 => { + Status::failed_precondition(format!("{resource} does not exist")) + } + kube::Error::Api(response) if response.code == 403 => Status::failed_precondition(format!( + "gateway is forbidden from reading metadata for {resource} (Kubernetes 403); check gateway RBAC" + )), + kube::Error::Api(response) if response.code == 401 => Status::unavailable(format!( + "gateway authentication was rejected while reading metadata for {resource} (Kubernetes 401)" + )), + kube::Error::Api(response) => Status::unavailable(format!( + "Kubernetes API returned {} ({}) while reading metadata for {resource}", + response.code, response.reason + )), + kube::Error::RustlsTls(_) | kube::Error::TlsRequired => Status::unavailable(format!( + "Kubernetes API TLS failed while reading metadata for {resource}" + )), + kube::Error::Auth(_) => Status::unavailable(format!( + "Kubernetes client authentication failed while reading metadata for {resource}" + )), + kube::Error::HyperError(_) | kube::Error::Service(_) => Status::unavailable(format!( + "Kubernetes API connection failed while reading metadata for {resource}" + )), + _ => Status::unavailable(format!( + "Kubernetes client failed while reading metadata for {resource}" + )), + } +} + fn reference(refs: &mut BTreeSet, kind: &'static str, name: Option<&str>, scope: Scope) { if let Some(name) = name.filter(|name| !name.is_empty()) { refs.insert(Reference { @@ -180,34 +222,39 @@ pub async fn admit( } else { Api::namespaced_with(client.clone(), namespace, &resource) }; + let description = resource_description(&reference, namespace, cluster); let object = tokio::time::timeout( std::time::Duration::from_secs(30), api.get_metadata(&reference.name), ) .await - .map_err(|_| Status::unavailable("resource admission lookup timed out"))? - .map_err(|error| match error { - kube::Error::Api(response) if response.code == 404 => { - Status::failed_precondition("external resource not admitted") - } - _ => Status::unavailable("resource admission metadata lookup failed"), - })?; + .map_err(|_| { + Status::unavailable(format!( + "Kubernetes API timed out while reading metadata for {description}" + )) + })? + .map_err(|error| metadata_lookup_error(error, &description))?; let metadata = object.metadata; if metadata.deletion_timestamp.is_some() { - return Err(Status::failed_precondition("resource is being deleted")); + return Err(Status::failed_precondition(format!( + "{description} is being deleted" + ))); } - let uid = metadata - .uid - .filter(|uid| !uid.is_empty()) - .ok_or_else(|| Status::failed_precondition("resource has no identity"))?; + let uid = metadata.uid.filter(|uid| !uid.is_empty()).ok_or_else(|| { + Status::failed_precondition(format!("{description} has no Kubernetes UID")) + })?; let labels = metadata .labels .as_ref() .into_iter() .flat_map(|labels| labels.iter()); match reference.scope { - Scope::Workspace => policy.admit(workspace, labels)?, - Scope::Shared => policy.admit_shared(labels)?, + Scope::Workspace => policy + .admit(workspace, labels) + .map_err(|status| contextualize(status, &description))?, + Scope::Shared => policy + .admit_shared(labels) + .map_err(|status| contextualize(status, &description))?, } identities.insert( format!( @@ -312,6 +359,125 @@ mod tests { } } } + + #[tokio::test] + async fn metadata_lookup_errors_identify_the_resource_and_failure_class() { + for (http_status, reason, expected_code, expected_message) in [ + ( + 404, + "NotFound", + tonic::Code::FailedPrecondition, + "does not exist", + ), + ( + 403, + "Forbidden", + tonic::Code::FailedPrecondition, + "check gateway RBAC", + ), + ( + 503, + "ServiceUnavailable", + tonic::Code::Unavailable, + "Kubernetes API returned 503 (ServiceUnavailable)", + ), + ] { + let service = tower::service_fn( + move |_request: http::Request| async move { + let body = serde_json::json!({ + "apiVersion": "v1", + "kind": "Status", + "status": "Failure", + "message": "fixture failure", + "reason": reason, + "code": http_status, + }); + Ok::<_, std::convert::Infallible>( + http::Response::builder() + .status(http_status) + .header("content-type", "application/json") + .body(kube::client::Body::from(body.to_string().into_bytes())) + .unwrap(), + ) + }, + ); + let client = Client::new(service, "shared"); + let spec = serde_json::json!({ + "automountServiceAccountToken": false, + "volumes": [{ + "name": "data", + "persistentVolumeClaim": {"claimName": "team-data"} + }] + }); + + let error = admit( + &client, + &ResourceAdmissionConfig::default(), + "team-a", + "shared", + &spec, + "private", + ) + .await + .expect_err("lookup must fail"); + + assert_eq!(error.code(), expected_code); + assert!( + error + .message() + .contains("PersistentVolumeClaim 'shared/team-data'"), + "unexpected error: {error}" + ); + assert!( + error.message().contains(expected_message), + "unexpected error: {error}" + ); + } + } + + #[tokio::test] + async fn label_denial_identifies_the_resource() { + let service = tower::service_fn(|_request: http::Request| async move { + let body = serde_json::json!({ + "apiVersion": "meta.k8s.io/v1", + "kind": "PartialObjectMetadata", + "metadata": { + "name": "kata", + "uid": "runtime-class-uid", + "labels": {} + } + }); + Ok::<_, std::convert::Infallible>( + http::Response::builder() + .header("content-type", "application/json") + .body(kube::client::Body::from(body.to_string().into_bytes())) + .unwrap(), + ) + }); + let client = Client::new(service, "shared"); + let spec = serde_json::json!({ + "automountServiceAccountToken": false, + "runtimeClassName": "kata" + }); + + let error = admit( + &client, + &ResourceAdmissionConfig::default(), + "team-a", + "shared", + &spec, + "private", + ) + .await + .expect_err("unlabelled RuntimeClass must be denied"); + + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert_eq!( + error.message(), + "RuntimeClass 'kata': external resource not admitted by required labels" + ); + } + #[test] fn inventories_all_containers_and_reference_aliases() { let pod = serde_json::json!({"automountServiceAccountToken":false,"runtimeClassName":"r","priorityClassName":"p", diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index a8d852562d..335a9ef2e3 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -685,7 +685,10 @@ impl PodmanComputeDriver { .flat_map(|labels| labels.iter()), ) .map_err(|error| { - ComputeDriverError::Precondition(error.message().into()) + ComputeDriverError::Precondition(format!( + "podman volume '{volume}': {}", + error.message() + )) })?; if !self.config.enable_bind_mounts && podman_volume_is_bind_backed(&volume_info) { @@ -775,7 +778,12 @@ impl PodmanComputeDriver { .into_iter() .flat_map(|labels| labels.iter()), ) - .map_err(precondition)?; + .map_err(|error| { + ComputeDriverError::Precondition(format!( + "podman volume '{name}': {}", + error.message() + )) + })?; if !self.config.enable_bind_mounts && podman_volume_is_bind_backed(&volume) { return Err(ComputeDriverError::Precondition( @@ -2997,10 +3005,17 @@ mod tests { }); let mut sandbox = sandbox_with_volume_mount("existing"); sandbox.workspace = "team-a".into(); - assert_eq!( - driver.validate_sandbox_create(&sandbox).await.is_ok(), - allowed - ); + let result = driver.validate_sandbox_create(&sandbox).await; + if !allowed { + assert!( + result + .as_ref() + .unwrap_err() + .to_string() + .contains("podman volume 'existing'") + ); + } + assert_eq!(result.is_ok(), allowed); handle.await.unwrap(); assert!( requests diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 92838af148..20a9c03bef 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -1852,12 +1852,6 @@ impl VmDriver { record.process.is_some() || record.provisioning_task.is_some(), ) }; - let mut sandbox = read_sandbox_request(&state_dir.join(SANDBOX_REQUEST_FILE)) - .await - .map_err(|error| { - Status::failed_precondition(format!("read VM admission provenance: {error}")) - })?; - self.validate_sandbox(&sandbox)?; if already_running { let active_generation = tokio::fs::read_to_string(state_dir.join(HOST_BOUNDARY_GENERATION_FILE)) @@ -1876,6 +1870,14 @@ impl VmDriver { if launch_authentication.is_empty() { return Ok(()); } + } + let mut sandbox = read_sandbox_request(&state_dir.join(SANDBOX_REQUEST_FILE)) + .await + .map_err(|error| { + Status::failed_precondition(format!("read VM admission provenance: {error}")) + })?; + self.validate_sandbox(&sandbox)?; + if already_running { // The gateway keeps launch sessions in memory. A non-empty bundle // during startup recovery represents a new gateway session, so // restart the VM before installing it rather than leaving the old @@ -8679,6 +8681,53 @@ mod tests { assert_eq!(persisted_authentication.supervisor.session_id, old_session); } + #[tokio::test] + async fn already_running_start_noop_does_not_require_admission_provenance() { + let temp = tempfile::tempdir().unwrap(); + let mut driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); + driver.config.state_dir = temp.path().to_path_buf(); + let sandbox = Sandbox { + id: "sandbox-running".to_string(), + name: "running".to_string(), + ..Default::default() + }; + let state_dir = temp.path().join("sandboxes").join(&sandbox.id); + create_private_dir_all(&state_dir).await.unwrap(); + tokio::fs::write( + state_dir.join(HOST_BOUNDARY_GENERATION_FILE), + b"g0000000000000001\n", + ) + .await + .unwrap(); + let provisioning_task = tokio::spawn(std::future::pending()); + let snapshot = sandbox_snapshot(&sandbox, provisioning_condition(), false); + driver.registry.lock().await.insert( + sandbox.id.clone(), + SandboxRecord { + snapshot, + state_dir, + process: None, + provisioning_task: Some(provisioning_task), + gpu_bdf: None, + deleting: false, + }, + ); + + driver + .start_sandbox(&sandbox.id, &sandbox.name, "g0000000000000001", Vec::new()) + .await + .expect("matching already-running start must remain an idempotent no-op"); + + let task = driver + .registry + .lock() + .await + .remove(&sandbox.id) + .and_then(|record| record.provisioning_task) + .unwrap(); + task.abort(); + } + fn test_launch_authentication(label: &str) -> (Vec, openshell_core::SandboxSessionId) { use openshell_core::jwt::{ SandboxLaunchAuthentication, SecretJwt, SessionVerificationKey, SupervisorAuthBundle, From 5181c52e845efbfa25b0e6a2f93d12292ce83071 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 22 Sep 2026 12:00:01 -0700 Subject: [PATCH 6/8] test(e2e): configure resource admission fixtures Signed-off-by: Drew Newberry --- .../openshell-driver-kubernetes/src/driver.rs | 70 ++++++++++++++++--- e2e/support/podman-gateway-config.sh | 3 + e2e/with-docker-gateway.sh | 3 + e2e/with-kube-gateway.sh | 3 + 4 files changed, 71 insertions(+), 8 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 9388012fb4..70e63f8daf 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1671,20 +1671,20 @@ impl KubernetesComputeDriver { .namespace .as_deref() .ok_or_else(|| tonic::Status::failed_precondition("sandbox lacks namespace"))?; - let sandbox_id = - sandbox_id_from_object(object).map_err(tonic::Status::failed_precondition)?; - let generation = sandbox_runtime_generation(object).ok_or_else(|| { - tonic::Status::failed_precondition("sandbox lacks runtime generation") + sandbox_id_from_object(object).map_err(tonic::Status::failed_precondition)?; + let spec = &object.data["spec"]["podTemplate"]["spec"]; + let private_secret = sandbox_bootstrap_secret_name(spec).ok_or_else(|| { + tonic::Status::failed_precondition( + "sandbox pod template is missing its bootstrap Secret volume", + ) })?; - let names = SandboxRuntimeNames::for_generation(&sandbox_id, generation); - let spec = object.data["spec"]["podTemplate"]["spec"].clone(); let actual = crate::resource_admission::admit( &self.client, &self.config.resource_admission, workspace, namespace, - &spec, - &names.sandbox_secret, + spec, + private_secret, ) .await?; if actual != expected { @@ -5654,6 +5654,15 @@ const SANDBOX_PROXY_CA_VOLUME_NAME: &str = "openshell-run"; const SANDBOX_PROXY_CA_MOUNT_PATH: &str = "/run"; const SANDBOX_BOOTSTRAP_SCHEDULING_GATE: &str = "openshell.ai/bootstrap"; +fn sandbox_bootstrap_secret_name(spec: &serde_json::Value) -> Option<&str> { + spec["volumes"] + .as_array()? + .iter() + .find(|volume| volume["name"] == SANDBOX_BOOTSTRAP_VOLUME_NAME)?["secret"]["secretName"] + .as_str() + .filter(|name| !name.is_empty()) +} + /// Render the workload Pod that runs the `OpenShell` sandbox runtime. /// /// The pod receives no gateway credential or endpoint. Its non-root sandbox @@ -7354,6 +7363,51 @@ mod tests { .is_empty() ); } + + #[tokio::test] + async fn admission_revalidates_stopped_sandbox_without_runtime_generation() { + let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig::default()); + let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( + SANDBOX_GROUP, + SANDBOX_VERSION_V1BETA1, + SANDBOX_KIND, + )); + let mut sandbox = DynamicObject::new("stopped-sandbox", &resource); + sandbox.metadata.namespace = Some("openshell".to_string()); + sandbox.metadata.labels = Some(BTreeMap::from([ + (LABEL_SANDBOX_ID.to_string(), "sandbox-id".to_string()), + (LABEL_SANDBOX_WORKSPACE.to_string(), "team-a".to_string()), + ])); + sandbox.metadata.annotations = Some(BTreeMap::from([ + ( + crate::resource_admission::CONFIG_USED.to_string(), + "false".to_string(), + ), + ( + crate::resource_admission::IDENTITIES.to_string(), + "{}".to_string(), + ), + ])); + sandbox.data = serde_json::json!({ + "spec": { + "podTemplate": { + "spec": { + "automountServiceAccountToken": false, + "volumes": [{ + "name": SANDBOX_BOOTSTRAP_VOLUME_NAME, + "secret": {"secretName": "os-sandbox-sandbox-id-oldgeneration"} + }] + } + } + } + }); + + driver + .admit_stored_resources(&sandbox) + .await + .expect("stopped sandbox should use its stored private Secret reference"); + } + use openshell_core::progress::{ PROGRESS_ACTIVE_DETAIL_KEY, PROGRESS_ACTIVE_STEP_KEY, PROGRESS_COMPLETE_LABEL_KEY, PROGRESS_COMPLETE_STEP_KEY, diff --git a/e2e/support/podman-gateway-config.sh b/e2e/support/podman-gateway-config.sh index 1f604ebcf7..5f7dffe3c6 100755 --- a/e2e/support/podman-gateway-config.sh +++ b/e2e/support/podman-gateway-config.sh @@ -148,6 +148,7 @@ e2e_write_podman_gateway_config() { done <"${output}" >"${configured_with_tls}" mv "${configured_with_tls}" "${output}" { + printf 'allow_driver_config = true\n' if [ "${external_driver}" = "1" ]; then printf 'socket_path = %s\n' "$(e2e_podman_toml_string "${driver_socket}")" else @@ -169,6 +170,8 @@ e2e_write_podman_gateway_config() { printf 'socket_path = %s\n' "$(e2e_podman_toml_string "${podman_socket}")" fi fi + printf '\n[openshell.drivers.podman.resource_admission]\n' + printf 'enabled = false\n' e2e_write_gateway_jwt_config "${jwt_dir}" "${gateway_id}" if [ "${oidc_mode}" != "1" ]; then e2e_write_gateway_mtls_auth_config diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index e4d0d8128f..84b1255dc0 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -596,6 +596,7 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" fi fi printf '[openshell.drivers.docker]\n' + printf 'allow_driver_config = true\n' if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")" else @@ -607,6 +608,8 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" printf 'sandbox_runtime_image = %s\n' "$(toml_string "${SANDBOX_RUNTIME_IMAGE}")" printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" fi + printf '\n[openshell.drivers.docker.resource_admission]\n' + printf 'enabled = false\n' } > "${GATEWAY_CONFIG}" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 36c321f071..926f1581df 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -1316,6 +1316,9 @@ else --docker-server=registry.example.test \ --docker-username=e2e-user \ --docker-password=e2e-password + kctl -n "${NAMESPACE}" label secret \ + "${OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET}" \ + openshell.ai/sandbox-attachable=true fi if [ "${OPENSHIFT_DETECTED}" = "1" ]; then From 62a34a39ce8b4fdc382815866951897326b8f538 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 22 Sep 2026 12:04:47 -0700 Subject: [PATCH 7/8] fix(kubernetes): retry forbidden admission lookups Signed-off-by: Drew Newberry --- .../openshell-driver-kubernetes/src/driver.rs | 98 +++++++++++++++++++ .../src/resource_admission.rs | 4 +- 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 70e63f8daf..dc64d1edb0 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -7408,6 +7408,104 @@ mod tests { .expect("stopped sandbox should use its stored private Secret reference"); } + #[tokio::test] + async fn admission_reconcile_does_not_suspend_on_forbidden_metadata_lookup() { + let sandbox = serde_json::json!({ + "apiVersion": "agents.x-k8s.io/v1beta1", + "kind": "Sandbox", + "metadata": { + "name": "sandbox-cr", + "namespace": "openshell", + "resourceVersion": "42", + "labels": { + LABEL_SANDBOX_ID: "sandbox-id", + LABEL_SANDBOX_WORKSPACE: "team-a" + }, + "annotations": { + crate::resource_admission::CONFIG_USED: "false", + crate::resource_admission::IDENTITIES: + "{\"PersistentVolumeClaim/openshell/team-data\":\"pvc-uid\"}" + } + }, + "spec": { + "podTemplate": { + "spec": { + "automountServiceAccountToken": false, + "volumes": [ + { + "name": "data", + "persistentVolumeClaim": {"claimName": "team-data"} + }, + { + "name": SANDBOX_BOOTSTRAP_VOLUME_NAME, + "secret": {"secretName": "os-sandbox-sandbox-id-generation"} + } + ] + } + } + } + }); + let steps = Arc::new(std::sync::Mutex::new(VecDeque::from([ + ( + http::Method::GET, + "/apis/agents.x-k8s.io/v1beta1/namespaces/openshell/sandboxes", + kube_test_response( + http::StatusCode::OK, + serde_json::json!({ + "apiVersion": "agents.x-k8s.io/v1beta1", + "kind": "SandboxList", + "items": [sandbox] + }), + ), + ), + ( + http::Method::GET, + "/api/v1/namespaces/openshell/persistentvolumeclaims/team-data", + kube_test_response( + http::StatusCode::FORBIDDEN, + serde_json::json!({ + "apiVersion": "v1", + "kind": "Status", + "status": "Failure", + "message": "fixture forbidden", + "reason": "Forbidden", + "code": 403 + }), + ), + ), + ]))); + let service_steps = steps.clone(); + let service = tower::service_fn(move |request: http::Request| { + let steps = service_steps.clone(); + async move { + let (method, path, response) = steps + .lock() + .unwrap() + .pop_front() + .expect("403 admission retry must not issue a suspension patch"); + assert_eq!(request.method(), method); + assert_eq!(request.uri().path(), path); + Ok::<_, std::convert::Infallible>(response) + } + }); + let client = Client::new(service, "openshell"); + let driver = KubernetesComputeDriver { + client: client.clone(), + watch_client: client, + sandbox_api_version: Arc::new(OnceCell::new()), + config: KubernetesComputeConfig::default(), + operator_allowlist: None, + }; + driver + .sandbox_api_version + .set(SANDBOX_VERSION_V1BETA1) + .expect("set test Sandbox API version"); + + driver.reconcile_sandbox_runtime_resources().await; + + assert!(steps.lock().unwrap().is_empty()); + } + use openshell_core::progress::{ PROGRESS_ACTIVE_DETAIL_KEY, PROGRESS_ACTIVE_STEP_KEY, PROGRESS_COMPLETE_LABEL_KEY, PROGRESS_COMPLETE_STEP_KEY, diff --git a/crates/openshell-driver-kubernetes/src/resource_admission.rs b/crates/openshell-driver-kubernetes/src/resource_admission.rs index d19d9c9e0a..fa288c4f80 100644 --- a/crates/openshell-driver-kubernetes/src/resource_admission.rs +++ b/crates/openshell-driver-kubernetes/src/resource_admission.rs @@ -49,7 +49,7 @@ fn metadata_lookup_error(error: kube::Error, resource: &str) -> Status { kube::Error::Api(response) if response.code == 404 => { Status::failed_precondition(format!("{resource} does not exist")) } - kube::Error::Api(response) if response.code == 403 => Status::failed_precondition(format!( + kube::Error::Api(response) if response.code == 403 => Status::unavailable(format!( "gateway is forbidden from reading metadata for {resource} (Kubernetes 403); check gateway RBAC" )), kube::Error::Api(response) if response.code == 401 => Status::unavailable(format!( @@ -372,7 +372,7 @@ mod tests { ( 403, "Forbidden", - tonic::Code::FailedPrecondition, + tonic::Code::Unavailable, "check gateway RBAC", ), ( From 0c6c60c1eb70b1c4bc8e8a4f6e06d74ea09069bc Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 22 Sep 2026 12:43:21 -0700 Subject: [PATCH 8/8] fix(e2e): preserve external driver admission defaults Signed-off-by: Drew Newberry --- e2e/parity/test.sh | 6 ++++++ e2e/support/podman-gateway-config.sh | 6 +++--- e2e/with-docker-gateway.sh | 6 +++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/e2e/parity/test.sh b/e2e/parity/test.sh index ad4ad65410..f1343f1cfb 100755 --- a/e2e/parity/test.sh +++ b/e2e/parity/test.sh @@ -59,6 +59,7 @@ source "${ROOT}/e2e/support/podman-gateway-config.sh" mkdir -p "${WORKDIR}/pki/client" "${WORKDIR}/jwt" e2e_write_podman_gateway_config "${WORKDIR}/v1.toml" 1 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test '' '' 0 '' e2e_write_podman_gateway_config "${WORKDIR}/v2.toml" 2 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test '' '' 0 '' +e2e_write_podman_gateway_config "${WORKDIR}/v2-external.toml" 2 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 1 socket network 18181 image:test 15 supervisor:test '' '' 0 '' assert_contains "${WORKDIR}/v1.toml" 'version = 1' assert_contains "${WORKDIR}/v1.toml" 'compute_drivers = ["podman"]' assert_contains "${WORKDIR}/v1.toml" 'image_pull_policy = "missing"' @@ -67,7 +68,12 @@ assert_contains "${WORKDIR}/v1.toml" 'guest_tls_ca = ' assert_contains "${WORKDIR}/v2.toml" 'version = 2' assert_contains "${WORKDIR}/v2.toml" 'compute_driver = "podman"' assert_contains "${WORKDIR}/v2.toml" 'image_pull_policy = "if_not_present"' +assert_contains "${WORKDIR}/v2.toml" 'allow_driver_config = true' +assert_contains "${WORKDIR}/v2.toml" '[openshell.drivers.podman.resource_admission]' assert_not_contains "${WORKDIR}/v2.toml" 'health_check_interval_secs = 0' +assert_contains "${WORKDIR}/v2-external.toml" 'socket_path = "socket"' +assert_not_contains "${WORKDIR}/v2-external.toml" 'allow_driver_config = true' +assert_not_contains "${WORKDIR}/v2-external.toml" '[openshell.drivers.podman.resource_admission]' # V2 guest TLS is emitted before its driver table; V1 is driver-local. OPENSHELL_E2E_PODMAN_OPTION_PROFILE=podman-options e2e_write_podman_gateway_config "${WORKDIR}/v1-options.toml" 1 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test "" "" 0 "" OPENSHELL_E2E_PODMAN_OPTION_PROFILE=podman-options e2e_write_podman_gateway_config "${WORKDIR}/v2-options.toml" 2 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test "" "" 0 "" diff --git a/e2e/support/podman-gateway-config.sh b/e2e/support/podman-gateway-config.sh index 5f7dffe3c6..9d6ea929e1 100755 --- a/e2e/support/podman-gateway-config.sh +++ b/e2e/support/podman-gateway-config.sh @@ -148,10 +148,10 @@ e2e_write_podman_gateway_config() { done <"${output}" >"${configured_with_tls}" mv "${configured_with_tls}" "${output}" { - printf 'allow_driver_config = true\n' if [ "${external_driver}" = "1" ]; then printf 'socket_path = %s\n' "$(e2e_podman_toml_string "${driver_socket}")" else + printf 'allow_driver_config = true\n' printf 'network_name = %s\n' "$(e2e_podman_toml_string "${network_name}")" printf 'gateway_port = %s\n' "${gateway_port}" printf 'default_image = %s\n' "$(e2e_podman_toml_string "${sandbox_image}")" @@ -169,9 +169,9 @@ e2e_write_podman_gateway_config() { if [ -n "${podman_socket}" ]; then printf 'socket_path = %s\n' "$(e2e_podman_toml_string "${podman_socket}")" fi + printf '\n[openshell.drivers.podman.resource_admission]\n' + printf 'enabled = false\n' fi - printf '\n[openshell.drivers.podman.resource_admission]\n' - printf 'enabled = false\n' e2e_write_gateway_jwt_config "${jwt_dir}" "${gateway_id}" if [ "${oidc_mode}" != "1" ]; then e2e_write_gateway_mtls_auth_config diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index 84b1255dc0..c4b5a446ed 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -596,10 +596,10 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" fi fi printf '[openshell.drivers.docker]\n' - printf 'allow_driver_config = true\n' if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")" else + printf 'allow_driver_config = true\n' printf 'sandbox_label = %s\n' "$(toml_string "${E2E_NAMESPACE}")" printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" @@ -607,9 +607,9 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" printf 'enable_bind_mounts = true\n' printf 'sandbox_runtime_image = %s\n' "$(toml_string "${SANDBOX_RUNTIME_IMAGE}")" printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" + printf '\n[openshell.drivers.docker.resource_admission]\n' + printf 'enabled = false\n' fi - printf '\n[openshell.drivers.docker.resource_admission]\n' - printf 'enabled = false\n' } > "${GATEWAY_CONFIG}" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then