diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index afd9cc2f2..8e51a7393 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -87,7 +87,7 @@ journalctl -u --no-pager --lines=200 journalctl -u openshell-gateway --no-pager --lines=200 ``` -Custom names use `[openshell.drivers.].socket_path`. A launch-time `--compute-driver-socket` override may also use `docker`, `podman`, `kubernetes`, or `vm`; the endpoint then takes precedence over built-in construction. The socket must be accessible only to the intended gateway identity. Check gateway logs for connection errors, `GetCapabilities` failures, or an unexpected advertised driver name. The advertised name is diagnostic metadata; negotiated features control optional behavior. The gateway does not create or supervise operator-supplied driver processes or sockets. +Custom names use `[openshell.drivers.].socket_path`. A launch-time `--compute-driver-socket` override may also use `docker`, `podman`, `kubernetes`, or `vm`; the endpoint then takes precedence over built-in construction. First-party standalone drivers require the socket parent directory to be owned by the driver's effective UID, force its mode to `0700`, create the socket with mode `0600`, and accept only peers with that same UID. Check the parent and socket separately with `stat`; a gateway running under a different UID cannot connect even when filesystem permissions or group membership would otherwise allow it. Operator-supplied drivers must provide equivalent access control appropriate to their implementation. Check gateway logs for connection errors, `GetCapabilities` failures, or an unexpected advertised driver name. The advertised name is diagnostic metadata; negotiated features control optional behavior. The gateway does not create or supervise operator-supplied driver processes or sockets. For configured gateway interceptors, inspect `[[openshell.gateway.interceptors]]`, their Unix or network endpoints, and gateway startup logs: diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 37154c75d..2b4d9d5d4 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -196,6 +196,20 @@ jobs: e2e-task: e2e:kubernetes:workspace-managed cli-artifact-prefix: rust-binary-cli + kubernetes-external-driver-e2e: + needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-kubernetes-test.yml + with: + image-tag: ${{ github.sha }} + job-name: Kubernetes E2E (external compute driver) + e2e-task: e2e:kubernetes:external-driver + cli-artifact-prefix: rust-binary-cli + kubernetes-workspace-operator-e2e: needs: [pr_metadata, build-gateway, build-supervisor, build-cli] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' @@ -240,7 +254,7 @@ jobs: core-e2e-result: name: Core E2E result - needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] + needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' runs-on: ubuntu-latest steps: @@ -252,6 +266,7 @@ jobs: BUILD_DRIVER_VM_RESULT: ${{ needs.build-driver-vm-linux.result }} E2E_RESULT: ${{ needs.e2e.result }} KUBERNETES_E2E_RESULT: ${{ needs.kubernetes-e2e.result }} + KUBERNETES_EXTERNAL_DRIVER_E2E_RESULT: ${{ needs.kubernetes-external-driver-e2e.result }} KUBERNETES_WORKSPACE_MANAGED_E2E_RESULT: ${{ needs.kubernetes-workspace-managed-e2e.result }} KUBERNETES_WORKSPACE_OPERATOR_E2E_RESULT: ${{ needs.kubernetes-workspace-operator-e2e.result }} run: | @@ -264,6 +279,7 @@ jobs: "build-driver-vm-linux:$BUILD_DRIVER_VM_RESULT" \ "e2e:$E2E_RESULT" \ "kubernetes-e2e:$KUBERNETES_E2E_RESULT" \ + "kubernetes-external-driver-e2e:$KUBERNETES_EXTERNAL_DRIVER_E2E_RESULT" \ "kubernetes-workspace-managed-e2e:$KUBERNETES_WORKSPACE_MANAGED_E2E_RESULT" \ "kubernetes-workspace-operator-e2e:$KUBERNETES_WORKSPACE_OPERATOR_E2E_RESULT"; do name="${item%%:*}" diff --git a/.github/workflows/e2e-kubernetes-test.yml b/.github/workflows/e2e-kubernetes-test.yml index bf13ab708..3e3657014 100644 --- a/.github/workflows/e2e-kubernetes-test.yml +++ b/.github/workflows/e2e-kubernetes-test.yml @@ -96,14 +96,17 @@ jobs: run: mise install --locked # The openshell-policy crate transitively pulls in z3-sys, whose - # build script needs the z3 C/C++ headers and clang/bindgen to - # compile. The bare runner doesn't ship them; the CI container + # build script needs the z3 C/C++ headers, clang/bindgen, and CMake to + # compile both system-linked and bundled-Z3 builds. The bare runner + # doesn't ship them; the CI container # image used by other Rust e2e jobs does, but we can't run this job # there (the runner's container handler injects its own --network # bridge, which conflicts with the --network host we need so kind's # API server is reachable from the test process). - name: Install z3 build deps - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libz3-dev clang + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends libz3-dev clang cmake - name: Log in to GHCR run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index c12379058..006f5cf44 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -59,6 +59,9 @@ jobs: - suite: rust-docker cmd: "mise run --no-deps --skip-deps e2e:rust" apt_packages: "openssh-client" + - suite: rust-docker-external-driver + cmd: "env -u OPENSHELL_GATEWAY_BIN mise run --no-deps --skip-deps e2e:docker:external-driver" + apt_packages: "openssh-client" - suite: mcp cmd: "mise run --no-deps --skip-deps e2e:mcp" apt_packages: "" @@ -127,7 +130,7 @@ jobs: run: ${{ matrix.cmd }} e2e-podman-rootless: - name: E2E (rust-podman-rootless, ${{ matrix.runner }}) + name: E2E (rust-podman-${{ matrix.suite }}, ${{ matrix.runner }}) # Run directly on the Ubuntu host so the test observes the host's AppArmor # and unprivileged-user-namespace policy. A privileged job container masks # the restrictions that production rootless Podman installations enforce. @@ -142,10 +145,18 @@ jobs: include: # Keep package versions explicit so hosted-runner tool overrides # cannot silently change the supported test environment. - - runner: ubuntu-26.04 + - suite: rootless + runner: ubuntu-26.04 podman_major: "5" podman_package_version: "5.7.0+ds2-3build1" conmon_package_version: "2.1.13+ds1-2" + cmd: "mise run --no-deps --skip-deps e2e:podman:rootless" + - suite: external-driver + runner: ubuntu-26.04 + podman_major: "5" + podman_package_version: "5.7.0+ds2-3build1" + conmon_package_version: "2.1.13+ds1-2" + cmd: "env -u OPENSHELL_GATEWAY_BIN mise run --no-deps --skip-deps e2e:podman:external-driver" env: IMAGE_TAG: ${{ inputs.image-tag }} MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -261,20 +272,28 @@ jobs: - name: Log in to GHCR with Podman run: echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin - - name: Run rootless Podman E2E - run: mise run --no-deps --skip-deps e2e:podman:rootless + - name: Run Podman E2E + run: ${{ matrix.cmd }} - name: Print AppArmor denials if: always() run: sudo dmesg | grep -E 'apparmor=.*DENIED|profile="unprivileged_userns"' | tail -100 || true e2e-vm: - name: E2E (rust-vm) + name: E2E (rust-vm-${{ matrix.suite }}) # libkrun needs KVM, so this job must run directly on a GitHub-hosted # Linux VM. GitHub-hosted macOS runners do not support nested # virtualization, and a job container would hide the host KVM device. runs-on: ubuntu-24.04 timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + include: + - suite: managed + cmd: "mise run --no-deps --skip-deps e2e:vm" + - suite: external-driver + cmd: "env -u OPENSHELL_GATEWAY_BIN mise run --no-deps --skip-deps e2e:vm:external-driver" env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -362,4 +381,4 @@ jobs: cache-on-failure: "true" - name: Run VM E2E - run: mise run --no-deps --skip-deps e2e:vm + run: ${{ matrix.cmd }} diff --git a/Cargo.lock b/Cargo.lock index fe85d58a7..a9eee33af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3776,6 +3776,7 @@ dependencies = [ "prost-types", "protoc-bin-vendored", "reqwest 0.12.28", + "rustix 1.1.4", "serde", "serde_json", "tar", @@ -3814,7 +3815,9 @@ version = "0.0.0" dependencies = [ "bollard", "bytes", + "clap", "futures", + "miette", "openshell-core", "prost-types", "serde", @@ -3824,8 +3827,10 @@ dependencies = [ "tempfile", "tokio", "tokio-stream", + "toml", "tonic", "tracing", + "tracing-subscriber", "url", ] diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 1fe0c37fe..5d4294286 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -120,6 +120,15 @@ to `run_cli_with_compute_drivers`; factories receive merged driver config and finish through the same in-process runtime adapter. A configured UDS endpoint still takes precedence over a compiled registration with the same name. +The standard server crate groups first-party registrations behind the +`in-tree-compute-drivers` feature. Protocol-only gateway builds disable that +feature and link no compute-driver crates. E2E lanes compose that gateway with +Docker, Podman, Kubernetes, and VM driver executables over the public UDS gRPC +contract so an in-tree driver cannot silently depend on a server-only API. +External Kubernetes drivers support shared and managed workspace modes. +Operator mode requires an in-process dynamic namespace allowlist and is +rejected when Kubernetes is configured through an external endpoint. + ## Stop and Start Lifecycle The gateway persists lifecycle intent before mutating compute: diff --git a/crates/openshell-core/Cargo.toml b/crates/openshell-core/Cargo.toml index 8b28fafa2..0e2ad24ff 100644 --- a/crates/openshell-core/Cargo.toml +++ b/crates/openshell-core/Cargo.toml @@ -35,6 +35,7 @@ tempfile = { version = "3", optional = true } [target.'cfg(unix)'.dependencies] nix = { workspace = true } +rustix = { workspace = true } [features] default = ["telemetry"] diff --git a/crates/openshell-core/src/external_driver_socket.rs b/crates/openshell-core/src/external_driver_socket.rs new file mode 100644 index 000000000..e6e1978f8 --- /dev/null +++ b/crates/openshell-core/src/external_driver_socket.rs @@ -0,0 +1,232 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Public Unix-socket transport helpers for out-of-process drivers. + +use std::io; +use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use tokio::net::{UnixListener, UnixStream}; +use tokio_stream::Stream; + +/// Prepare and bind a private Unix socket owned by the current effective UID. +pub fn bind_private(path: &Path) -> Result { + let parent = path + .parent() + .ok_or_else(|| format!("driver socket path '{}' has no parent", path.display()))?; + let expected_uid = rustix::process::geteuid().as_raw(); + std::fs::create_dir_all(parent) + .map_err(|err| format!("create socket directory {}: {err}", parent.display()))?; + let parent_metadata = std::fs::symlink_metadata(parent) + .map_err(|err| format!("stat socket directory {}: {err}", parent.display()))?; + if parent_metadata.file_type().is_symlink() || !parent_metadata.file_type().is_dir() { + return Err(format!( + "driver socket parent '{}' must be a directory, not a symlink", + parent.display() + )); + } + if parent_metadata.uid() != expected_uid { + return Err(format!( + "driver socket parent '{}' is owned by uid {}, expected {}", + parent.display(), + parent_metadata.uid(), + expected_uid + )); + } + std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700)) + .map_err(|err| format!("chmod socket directory {}: {err}", parent.display()))?; + + match std::fs::symlink_metadata(path) { + Ok(metadata) + if metadata.file_type().is_socket() + && !metadata.file_type().is_symlink() + && metadata.uid() == expected_uid => + { + std::fs::remove_file(path) + .map_err(|err| format!("remove stale socket {}: {err}", path.display()))?; + } + Ok(_) => { + return Err(format!( + "driver socket path '{}' exists but is not an owned Unix socket", + path.display() + )); + } + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + Err(err) => return Err(format!("stat driver socket {}: {err}", path.display())), + } + + let listener = UnixListener::bind(path) + .map_err(|err| format!("bind driver socket {}: {err}", path.display()))?; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .map_err(|err| format!("chmod driver socket {}: {err}", path.display()))?; + Ok(listener) +} + +/// Remove a socket created by [`bind_private`]. +pub struct SocketCleanup(PathBuf); + +impl SocketCleanup { + #[must_use] + pub fn new(path: PathBuf) -> Self { + Self(path) + } +} + +impl Drop for SocketCleanup { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} + +/// Incoming UDS connections restricted to the driver's effective UID. +pub struct SameUidUnixIncoming { + listener: UnixListener, + expected_uid: u32, +} + +impl SameUidUnixIncoming { + #[must_use] + pub fn new(listener: UnixListener) -> Self { + Self { + listener, + expected_uid: rustix::process::geteuid().as_raw(), + } + } +} + +impl Stream for SameUidUnixIncoming { + type Item = io::Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + loop { + match this.listener.poll_accept(cx) { + Poll::Ready(Ok((stream, _))) => match stream.peer_cred() { + Ok(credentials) if credentials.uid() == this.expected_uid => { + return Poll::Ready(Some(Ok(stream))); + } + Ok(credentials) => tracing::warn!( + peer_uid = credentials.uid(), + expected_uid = this.expected_uid, + "rejected external driver socket client" + ), + Err(err) => { + tracing::warn!(error = %err, "failed to authenticate driver socket client"); + } + }, + Poll::Ready(Err(err)) => return Poll::Ready(Some(Err(err))), + Poll::Pending => return Poll::Pending, + } + } + } +} + +#[cfg(test)] +mod tests { + use std::os::unix::fs::{PermissionsExt, symlink}; + + use tokio_stream::StreamExt; + + use super::*; + + #[tokio::test] + async fn bind_private_creates_private_socket_and_cleanup_removes_it() { + let temp_dir = tempfile::tempdir().expect("create temporary directory"); + let socket_dir = temp_dir.path().join("driver"); + let socket_path = socket_dir.join("compute.sock"); + + let listener = bind_private(&socket_path).expect("bind private socket"); + + let directory_mode = std::fs::metadata(&socket_dir) + .expect("stat socket directory") + .permissions() + .mode() + & 0o777; + let socket_mode = std::fs::metadata(&socket_path) + .expect("stat socket") + .permissions() + .mode() + & 0o777; + assert_eq!(directory_mode, 0o700); + assert_eq!(socket_mode, 0o600); + + drop(listener); + let cleanup = SocketCleanup::new(socket_path.clone()); + drop(cleanup); + assert!(!socket_path.exists()); + } + + #[test] + fn bind_private_rejects_symlinked_parent() { + let temp_dir = tempfile::tempdir().expect("create temporary directory"); + let actual_dir = temp_dir.path().join("actual"); + let linked_dir = temp_dir.path().join("linked"); + std::fs::create_dir(&actual_dir).expect("create socket directory"); + symlink(&actual_dir, &linked_dir).expect("create directory symlink"); + + let err = bind_private(&linked_dir.join("compute.sock")) + .expect_err("symlinked socket parent must be rejected"); + + assert!(err.contains("must be a directory, not a symlink")); + } + + #[test] + fn bind_private_rejects_existing_non_socket() { + let temp_dir = tempfile::tempdir().expect("create temporary directory"); + let socket_path = temp_dir.path().join("compute.sock"); + std::fs::write(&socket_path, b"not a socket").expect("create conflicting file"); + + let err = bind_private(&socket_path).expect_err("non-socket path must be rejected"); + + assert!(err.contains("is not an owned Unix socket")); + assert_eq!( + std::fs::read(&socket_path).expect("read conflicting file"), + b"not a socket" + ); + } + + #[tokio::test] + async fn bind_private_replaces_existing_owned_socket() { + let temp_dir = tempfile::tempdir().expect("create temporary directory"); + let socket_path = temp_dir.path().join("compute.sock"); + let stale_listener = + std::os::unix::net::UnixListener::bind(&socket_path).expect("bind stale Unix socket"); + drop(stale_listener); + + let listener = bind_private(&socket_path).expect("replace owned Unix socket"); + + assert!( + std::fs::symlink_metadata(&socket_path) + .expect("stat replacement socket") + .file_type() + .is_socket() + ); + drop(listener); + } + + #[tokio::test] + async fn same_uid_incoming_accepts_current_user() { + let temp_dir = tempfile::tempdir().expect("create temporary directory"); + let socket_path = temp_dir.path().join("compute.sock"); + let listener = bind_private(&socket_path).expect("bind private socket"); + let mut incoming = SameUidUnixIncoming::new(listener); + + let client = UnixStream::connect(&socket_path) + .await + .expect("connect to private socket"); + let accepted = incoming + .next() + .await + .expect("incoming stream ended") + .expect("accept same-UID client"); + + assert_eq!( + accepted.peer_cred().expect("read client credentials").uid(), + rustix::process::geteuid().as_raw() + ); + drop(client); + } +} diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index d373d656e..96be19e1e 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -18,6 +18,8 @@ pub mod driver_mounts; pub mod driver_utils; pub mod endpoint_path; pub mod error; +#[cfg(unix)] +pub mod external_driver_socket; pub mod forward; pub mod google_cloud; pub mod gpu; @@ -29,6 +31,7 @@ pub mod jwt; pub mod metadata; pub mod middleware; pub mod net; +pub mod operator_namespace_allowlist; pub mod paths; pub mod policy; pub mod progress; @@ -53,6 +56,7 @@ pub use error::{ComputeDriverError, Error, Result}; pub use metadata::{ GetResourceVersion, ObjectId, ObjectLabels, ObjectName, ObjectWorkspace, SetResourceVersion, }; +pub use operator_namespace_allowlist::OperatorNamespaceAllowlist; /// Build version string derived from git metadata. /// diff --git a/crates/openshell-core/src/operator_namespace_allowlist.rs b/crates/openshell-core/src/operator_namespace_allowlist.rs new file mode 100644 index 000000000..c8f0f7f3d --- /dev/null +++ b/crates/openshell-core/src/operator_namespace_allowlist.rs @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::BTreeSet; +use std::sync::{Arc, RwLock}; + +/// Thread-safe dynamic allowlist of Kubernetes operator-mode namespaces. +/// +/// This type lives in the public core API because both the Kubernetes driver +/// and gateway authentication boundary consume it. +#[derive(Debug, Clone)] +pub struct OperatorNamespaceAllowlist { + inner: Arc>>, +} + +impl OperatorNamespaceAllowlist { + fn read_guard(&self) -> std::sync::RwLockReadGuard<'_, BTreeSet> { + self.inner + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn write_guard(&self) -> std::sync::RwLockWriteGuard<'_, BTreeSet> { + self.inner + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + #[must_use] + pub fn new() -> Self { + Self { + inner: Arc::new(RwLock::new(BTreeSet::new())), + } + } + + #[must_use] + pub fn from_set(set: BTreeSet) -> Self { + Self { + inner: Arc::new(RwLock::new(set)), + } + } + + pub fn replace(&self, new_set: BTreeSet) { + *self.write_guard() = new_set; + } + + pub fn merge(&self, additional: &BTreeSet) { + self.write_guard().extend(additional.iter().cloned()); + } + + pub fn read(&self) -> std::sync::RwLockReadGuard<'_, BTreeSet> { + self.read_guard() + } + + #[must_use] + pub fn contains(&self, namespace: &str) -> bool { + self.read_guard().contains(namespace) + } + + pub fn insert(&self, name: String) -> bool { + self.write_guard().insert(name) + } + + pub fn remove(&self, name: &str) -> bool { + self.write_guard().remove(name) + } + + #[must_use] + pub fn shared(&self) -> Arc>> { + Arc::clone(&self.inner) + } +} + +impl Default for OperatorNamespaceAllowlist { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index 92a32c6d2..065b3ff4b 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -10,11 +10,15 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[[bin]] +name = "openshell-driver-docker" +path = "src/main.rs" + [dependencies] openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } tokio = { workspace = true } -tonic = { workspace = true } +tonic = { workspace = true, features = ["transport"] } futures = { workspace = true } tokio-stream = { workspace = true } tracing = { workspace = true } @@ -24,6 +28,10 @@ serde_json = { workspace = true } prost-types = { workspace = true } bollard = { version = "0.20" } url = { workspace = true } +clap = { workspace = true } +miette = { workspace = true } +toml = { workspace = true } +tracing-subscriber = { workspace = true } [dev-dependencies] prost-types = { workspace = true } diff --git a/crates/openshell-driver-docker/src/main.rs b/crates/openshell-driver-docker/src/main.rs new file mode 100644 index 000000000..3c539ba11 --- /dev/null +++ b/crates/openshell-driver-docker/src/main.rs @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::net::SocketAddr; +use std::path::PathBuf; + +use clap::Parser; +use miette::{IntoDiagnostic, Result}; +use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_core::{Config, VERSION}; +use openshell_driver_docker::{DockerComputeConfig, DockerComputeDriver}; +use tracing::info; +use tracing_subscriber::EnvFilter; + +#[derive(Debug, Parser)] +#[command(name = "openshell-driver-docker", version = VERSION)] +struct Args { + /// Public compute-driver Unix socket used by the gateway. + #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] + bind_socket: PathBuf, + + /// TOML file containing a serialized `DockerComputeConfig` table. + #[arg(long, env = "OPENSHELL_DOCKER_DRIVER_CONFIG")] + config: PathBuf, + + /// Gateway listener address used to derive sandbox callback routing. + #[arg( + long, + env = "OPENSHELL_GATEWAY_BIND", + default_value = "127.0.0.1:50051" + )] + gateway_bind: SocketAddr, + + #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] + log_level: String, +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), + ) + .init(); + + let config_source = std::fs::read_to_string(&args.config).into_diagnostic()?; + let docker_config: DockerComputeConfig = toml::from_str(&config_source).into_diagnostic()?; + let gateway_config = Config::new(None).with_bind_address(args.gateway_bind); + let driver = DockerComputeDriver::new(&gateway_config, &docker_config) + .await + .into_diagnostic()?; + + let listener = openshell_core::external_driver_socket::bind_private(&args.bind_socket) + .map_err(|err| miette::miette!("{err}"))?; + let _cleanup = + openshell_core::external_driver_socket::SocketCleanup::new(args.bind_socket.clone()); + info!(socket = %args.bind_socket.display(), "Starting Docker compute driver"); + tonic::transport::Server::builder() + .add_service(ComputeDriverServer::new(driver)) + .serve_with_incoming_shutdown( + openshell_core::external_driver_socket::SameUidUnixIncoming::new(listener), + shutdown_signal(), + ) + .await + .into_diagnostic() +} + +async fn shutdown_signal() { + let terminate = async { + match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) { + Ok(mut signal) => { + signal.recv().await; + } + Err(_) => std::future::pending::<()>().await, + } + }; + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + () = terminate => {} + } + info!("Received shutdown signal, draining in-flight requests"); +} diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 1cdc98f6e..aedd3b8bf 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -1,12 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +pub use openshell_core::OperatorNamespaceAllowlist; use openshell_core::config; use serde::{Deserialize, Deserializer, Serialize}; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; +#[cfg(test)] +use std::collections::BTreeSet; use std::path::Path; use std::str::FromStr; -use std::sync::{Arc, RwLock}; /// Default gateway identity used in managed-mode namespace naming. pub const DEFAULT_GATEWAY_ID: &str = "openshell"; @@ -841,89 +843,6 @@ pub fn validate_managed_namespace_name(gateway_id: &str, workspace: &str) -> Res Ok(()) } -/// Thread-safe dynamic allowlist of valid operator-mode namespaces. -/// -/// Backed by an `Arc>>` that is updated by background -/// tasks (label selector watcher, drop-in file watcher) and read by the SA -/// authenticator and namespace resolver. -#[derive(Debug, Clone)] -pub struct OperatorNamespaceAllowlist { - inner: Arc>>, -} - -impl OperatorNamespaceAllowlist { - fn read_guard(&self) -> std::sync::RwLockReadGuard<'_, BTreeSet> { - self.inner - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner) - } - - fn write_guard(&self) -> std::sync::RwLockWriteGuard<'_, BTreeSet> { - self.inner - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner) - } - - #[must_use] - pub fn new() -> Self { - Self { - inner: Arc::new(RwLock::new(BTreeSet::new())), - } - } - - #[must_use] - pub fn from_set(set: BTreeSet) -> Self { - Self { - inner: Arc::new(RwLock::new(set)), - } - } - - /// Replace the entire allowlist (used by background watchers on refresh). - pub fn replace(&self, new_set: BTreeSet) { - let mut guard = self.write_guard(); - *guard = new_set; - } - - /// Merge additional namespaces into the allowlist. - pub fn merge(&self, additional: &BTreeSet) { - let mut guard = self.write_guard(); - guard.extend(additional.iter().cloned()); - } - - /// Read the current allowlist snapshot. - pub fn read(&self) -> std::sync::RwLockReadGuard<'_, BTreeSet> { - self.read_guard() - } - - /// Check whether a namespace is in the allowlist. - #[must_use] - pub fn contains(&self, namespace: &str) -> bool { - self.read_guard().contains(namespace) - } - - /// Insert a namespace into the allowlist. Returns `true` if it was new. - pub fn insert(&self, name: String) -> bool { - self.write_guard().insert(name) - } - - /// Remove a namespace from the allowlist. Returns `true` if it was present. - pub fn remove(&self, name: &str) -> bool { - self.write_guard().remove(name) - } - - /// Return a clone of the inner `Arc` for sharing with background tasks. - #[must_use] - pub fn shared(&self) -> Arc>> { - Arc::clone(&self.inner) - } -} - -impl Default for OperatorNamespaceAllowlist { - fn default() -> Self { - Self::new() - } -} - fn is_dns1123_subdomain(value: &str) -> bool { !value.is_empty() && value.len() <= 253 diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index 1a234385c..d69f9749a 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -8,8 +8,9 @@ pub mod grpc; pub use config::{ AppArmorProfile, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesSidecarConfig, - ManagedSshIngressConfig, OperatorNamespaceAllowlist, SupervisorSideloadMethod, - SupervisorTopology, WorkspaceMode, managed_namespace_prefix, + ManagedSshIngressConfig, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, + managed_namespace_prefix, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; +pub use openshell_core::OperatorNamespaceAllowlist; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 30b4fcada..3a805c868 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -5,6 +5,7 @@ use clap::{ArgAction, Parser}; use miette::{IntoDiagnostic, Result}; use std::collections::BTreeMap; use std::net::SocketAddr; +use std::path::PathBuf; use tracing::info; use tracing_subscriber::EnvFilter; @@ -22,6 +23,10 @@ use openshell_driver_kubernetes::{ #[command(version = VERSION)] #[allow(clippy::struct_excessive_bools)] struct Args { + /// Public compute-driver Unix socket used by an external gateway. + #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] + bind_socket: Option, + #[arg( long, env = "OPENSHELL_COMPUTE_DRIVER_BIND", @@ -286,13 +291,31 @@ async fn main() -> Result<()> { .await .into_diagnostic()?; - info!(address = %args.bind_address, "Starting Kubernetes compute driver"); - tonic::transport::Server::builder() - .add_service(ComputeDriverServer::new(ComputeDriverService::new(driver))) - .serve_with_shutdown(args.bind_address, async move { - shutdown_signal().await; - let _ = shutdown_tx.send(true); - }) - .await - .into_diagnostic() + let service = ComputeDriverServer::new(ComputeDriverService::new(driver)); + let shutdown = async move { + shutdown_signal().await; + let _ = shutdown_tx.send(true); + }; + if let Some(socket_path) = args.bind_socket { + let listener = openshell_core::external_driver_socket::bind_private(&socket_path) + .map_err(|err| miette::miette!("{err}"))?; + let _cleanup = + openshell_core::external_driver_socket::SocketCleanup::new(socket_path.clone()); + info!(socket = %socket_path.display(), "Starting Kubernetes compute driver"); + tonic::transport::Server::builder() + .add_service(service) + .serve_with_incoming_shutdown( + openshell_core::external_driver_socket::SameUidUnixIncoming::new(listener), + shutdown, + ) + .await + .into_diagnostic() + } else { + info!(address = %args.bind_address, "Starting Kubernetes compute driver"); + tonic::transport::Server::builder() + .add_service(service) + .serve_with_shutdown(args.bind_address, shutdown) + .await + .into_diagnostic() + } } diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 81d4254d1..1eb1eb257 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -23,6 +23,10 @@ use openshell_driver_podman::{ComputeDriverService, PodmanComputeConfig, PodmanC #[command(name = "openshell-driver-podman")] #[command(version = VERSION)] struct Args { + /// Public compute-driver Unix socket used by an external gateway. + #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] + bind_socket: Option, + #[arg( long, env = "OPENSHELL_COMPUTE_DRIVER_BIND", @@ -154,6 +158,10 @@ struct Args { /// Each entry is `"container_id:host_id:size"`. #[arg(long = "gidmap")] gidmap: Vec, + + /// Allow sandbox requests to attach host bind mounts. + #[arg(long, env = "OPENSHELL_ENABLE_BIND_MOUNTS", default_value_t = false)] + enable_bind_mounts: bool, } #[tokio::main] @@ -203,21 +211,37 @@ async fn main() -> Result<()> { userns: args.userns, uidmap: args.uidmap, gidmap: args.gidmap, + enable_bind_mounts: args.enable_bind_mounts, ..PodmanComputeConfig::default() }) .await .into_diagnostic()?; - info!(address = %args.bind_address, "Starting Podman compute driver"); - let result = tonic::transport::Server::builder() - .layer(compute_driver_rpc_layer()) - .add_service(ComputeDriverServer::new(ComputeDriverService::new(driver))) - .serve_with_shutdown(args.bind_address, async { - shutdown_signal().await; - info!("Received shutdown signal, draining in-flight requests"); - }) - .await - .into_diagnostic(); + let service = ComputeDriverServer::new(ComputeDriverService::new(driver)); + let result = if let Some(socket_path) = args.bind_socket { + let listener = openshell_core::external_driver_socket::bind_private(&socket_path) + .map_err(|err| miette::miette!("{err}"))?; + let _cleanup = + openshell_core::external_driver_socket::SocketCleanup::new(socket_path.clone()); + info!(socket = %socket_path.display(), "Starting Podman compute driver"); + tonic::transport::Server::builder() + .layer(compute_driver_rpc_layer()) + .add_service(service) + .serve_with_incoming_shutdown( + openshell_core::external_driver_socket::SameUidUnixIncoming::new(listener), + shutdown_signal(), + ) + .await + .into_diagnostic() + } else { + info!(address = %args.bind_address, "Starting Podman compute driver"); + tonic::transport::Server::builder() + .layer(compute_driver_rpc_layer()) + .add_service(service) + .serve_with_shutdown(args.bind_address, shutdown_signal()) + .await + .into_diagnostic() + }; if let Some(provider) = &tracer_provider && let Err(error) = provider.shutdown() { @@ -260,6 +284,8 @@ async fn shutdown_signal() { #[cfg(not(unix))] ctrl_c_signal().await; + + info!("Received shutdown signal, draining in-flight requests"); } #[cfg(test)] diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 772590d1b..215f6abfa 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -115,12 +115,19 @@ arc-swap = "1" notify = "8" [target.'cfg(not(target_os = "windows"))'.dependencies] -openshell-driver-docker = { path = "../openshell-driver-docker" } -openshell-driver-kubernetes = { path = "../openshell-driver-kubernetes" } -openshell-driver-podman = { path = "../openshell-driver-podman" } +openshell-driver-docker = { path = "../openshell-driver-docker", optional = true } +openshell-driver-kubernetes = { path = "../openshell-driver-kubernetes", optional = true } +openshell-driver-podman = { path = "../openshell-driver-podman", optional = true } [features] -default = ["telemetry"] +default = ["telemetry", "in-tree-compute-drivers"] +## Link the first-party compute drivers into the standard gateway binary. +## Disable this feature for a protocol-only gateway that uses external drivers. +in-tree-compute-drivers = [ + "dep:openshell-driver-docker", + "dep:openshell-driver-kubernetes", + "dep:openshell-driver-podman", +] ## Compile in anonymous telemetry emission (forwards to openshell-core/telemetry). ## On by default; build with `--no-default-features` for a telemetry-free gateway ## that contains no telemetry endpoint, HTTP client, or emission code. diff --git a/crates/openshell-server/src/auth/k8s_sa.rs b/crates/openshell-server/src/auth/k8s_sa.rs index 32cb2e119..131dbaba4 100644 --- a/crates/openshell-server/src/auth/k8s_sa.rs +++ b/crates/openshell-server/src/auth/k8s_sa.rs @@ -26,7 +26,7 @@ use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use kube::Error as KubeError; use kube::api::{Api, ApiResource, PostParams}; use kube::core::{DynamicObject, gvk::GroupVersionKind}; -use openshell_driver_kubernetes::OperatorNamespaceAllowlist; +use openshell_core::OperatorNamespaceAllowlist; use std::sync::Arc; use tonic::Status; use tracing::{debug, info, warn}; diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index f0eb6f98b..fad71efd3 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -7,7 +7,7 @@ //! driver-specific environment overrides, and applying gateway startup defaults. //! It does not acquire, connect to, or start compute drivers. -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] pub mod builtin; use crate::config_file; @@ -53,19 +53,73 @@ pub fn remote_driver_config_from_context( context: DriverStartupContext<'_>, name: &str, ) -> Result { - let mut cfg = driver_config_from_context(context, name)?; + let mut cfg = RemoteDriverConfig::default(); + if let Some(file) = context.file { + let merged = config_file::driver_table( + name, + &file.openshell.gateway, + file.openshell.drivers.get(name), + ); + if let Some(socket_path) = merged.get("socket_path").and_then(toml::Value::as_str) { + cfg.socket_path = PathBuf::from(socket_path); + } + } apply_remote_driver_overrides(&mut cfg, context, name); validate_remote_driver_config(&cfg, name)?; Ok(cfg) } #[derive(Debug, Clone, Default, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] pub struct RemoteDriverConfig { #[serde(default)] pub socket_path: PathBuf, } +#[derive(Debug, Clone, Deserialize)] +#[serde(default)] +pub struct KubernetesSaBootstrapConfig { + pub namespace: String, + pub service_account_name: String, + pub workspace_mode: String, + pub gateway_id: String, +} + +impl Default for KubernetesSaBootstrapConfig { + fn default() -> Self { + Self { + namespace: "openshell".to_string(), + service_account_name: "default".to_string(), + workspace_mode: "shared".to_string(), + gateway_id: "openshell".to_string(), + } + } +} + +pub fn kubernetes_sa_bootstrap_config( + file: Option<&config_file::ConfigFile>, +) -> Result { + let Some(file) = file else { + return Err(Error::config( + "K8s ServiceAccount bootstrap requires [openshell.drivers.kubernetes] when sandbox JWT issuing is enabled in-cluster", + )); + }; + if !file.openshell.drivers.contains_key("kubernetes") { + return Err(Error::config( + "K8s ServiceAccount bootstrap requires [openshell.drivers.kubernetes] when sandbox JWT issuing is enabled in-cluster", + )); + } + let merged = config_file::driver_table( + "kubernetes", + &file.openshell.gateway, + file.openshell.drivers.get("kubernetes"), + ); + merged.try_into().map_err(|error| { + Error::config(format!( + "invalid Kubernetes ServiceAccount bootstrap config: {error}" + )) + }) +} + pub fn driver_config_from_context( context: DriverStartupContext<'_>, driver_name: &str, @@ -157,6 +211,52 @@ socket_path = "/run/openshell/kyma.sock" assert_eq!(cfg.socket_path, PathBuf::from("/run/openshell/kyma.sock")); } + #[test] + fn remote_driver_config_ignores_in_process_driver_fields() { + let file: config_file::ConfigFile = toml::from_str( + r#" +[openshell.gateway] +sandbox_namespace = "sandboxes" + +[openshell.drivers.kubernetes] +socket_path = "/run/openshell/kubernetes.sock" +workspace_mode = "shared" +service_account_name = "sandbox-sa" +"#, + ) + .expect("valid config"); + + let cfg = remote_driver_config_from_context(test_context(Some(&file)), "kubernetes") + .expect("remote config"); + assert_eq!( + cfg.socket_path, + PathBuf::from("/run/openshell/kubernetes.sock") + ); + } + + #[test] + fn kubernetes_sa_bootstrap_uses_public_gateway_config() { + let file: config_file::ConfigFile = toml::from_str( + r#" +[openshell.gateway] +sandbox_namespace = "sandboxes" + +[openshell.drivers.kubernetes] +socket_path = "/run/openshell/kubernetes.sock" +workspace_mode = "managed" +gateway_id = "gateway-a" +service_account_name = "sandbox-sa" +"#, + ) + .expect("valid config"); + + let cfg = kubernetes_sa_bootstrap_config(Some(&file)).expect("bootstrap config"); + assert_eq!(cfg.namespace, "sandboxes"); + assert_eq!(cfg.workspace_mode, "managed"); + assert_eq!(cfg.gateway_id, "gateway-a"); + assert_eq!(cfg.service_account_name, "sandbox-sa"); + } + #[test] fn remote_driver_config_uses_endpoint_override_without_file() { let endpoint_overrides = diff --git a/crates/openshell-server/src/compute/driver_config/builtin.rs b/crates/openshell-server/src/compute/driver_config/builtin.rs index 948c982dd..dea867237 100644 --- a/crates/openshell-server/src/compute/driver_config/builtin.rs +++ b/crates/openshell-server/src/compute/driver_config/builtin.rs @@ -3,12 +3,11 @@ //! Configuration construction for built-in compute drivers. -use super::{ - DriverStartupContext, GuestTlsPaths, driver_config_from_context, driver_config_from_file, -}; +use super::{DriverStartupContext, GuestTlsPaths, driver_config_from_context}; use crate::compute::VmComputeConfig; +#[cfg(test)] use crate::config_file; -use openshell_core::{ComputeDriverKind, Error, Result}; +use openshell_core::{ComputeDriverKind, Result}; use openshell_driver_docker::DockerComputeConfig; use openshell_driver_kubernetes::KubernetesComputeConfig; use openshell_driver_podman::PodmanComputeConfig; @@ -23,22 +22,6 @@ pub fn kubernetes_config_from_context( Ok(cfg) } -pub fn kubernetes_config_for_k8s_sa_bootstrap( - file: Option<&config_file::ConfigFile>, -) -> Result { - let Some(file) = file else { - return Err(Error::config( - "K8s ServiceAccount bootstrap requires [openshell.drivers.kubernetes] when sandbox JWT issuing is enabled in-cluster", - )); - }; - if !file.openshell.drivers.contains_key("kubernetes") { - return Err(Error::config( - "K8s ServiceAccount bootstrap requires [openshell.drivers.kubernetes] when sandbox JWT issuing is enabled in-cluster", - )); - } - driver_config_from_file(Some(file), ComputeDriverKind::Kubernetes.as_str()) -} - /// Build the selected Podman config from TOML plus runtime defaults. pub fn podman_config_from_context( context: DriverStartupContext<'_>, @@ -165,35 +148,6 @@ mod tests { } } - #[test] - fn k8s_sa_bootstrap_rejects_missing_kubernetes_driver_config() { - let err = kubernetes_config_for_k8s_sa_bootstrap(None).unwrap_err(); - assert!(err.to_string().contains("[openshell.drivers.kubernetes]")); - - let file: config_file::ConfigFile = - toml::from_str("[openshell.gateway]\n").expect("valid config"); - let err = kubernetes_config_for_k8s_sa_bootstrap(Some(&file)).unwrap_err(); - assert!(err.to_string().contains("[openshell.drivers.kubernetes]")); - } - - #[test] - fn k8s_sa_bootstrap_uses_configured_namespace_and_service_account() { - let file: config_file::ConfigFile = toml::from_str( - r#" -[openshell.gateway] - -[openshell.drivers.kubernetes] -namespace = "sandboxes" -service_account_name = "sandbox-sa" -"#, - ) - .expect("valid config"); - - let cfg = kubernetes_config_for_k8s_sa_bootstrap(Some(&file)).unwrap(); - assert_eq!(cfg.namespace, "sandboxes"); - assert_eq!(cfg.service_account_name, "sandbox-sa"); - } - #[test] fn podman_config_reads_bind_mount_opt_in_from_driver_table() { let file: config_file::ConfigFile = toml::from_str( diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 7f1dc19a2..257508b74 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -5,16 +5,16 @@ pub mod driver_config; pub mod lease; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] pub mod vm; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] pub use openshell_driver_docker::DockerComputeConfig; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] pub use openshell_driver_kubernetes::KubernetesComputeConfig; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] pub use openshell_driver_podman::PodmanComputeConfig; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] pub use vm::VmComputeConfig; use crate::grpc::policy::SANDBOX_SETTINGS_OBJECT_TYPE; @@ -49,14 +49,14 @@ use openshell_core::proto::{ SandboxTemplate, ServiceEndpoint, SshSession, }; use openshell_core::{ObjectLabels, ObjectWorkspace}; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] use openshell_driver_docker::DockerComputeDriver; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] use openshell_driver_kubernetes::{ ComputeDriverService as KubernetesDriverService, KubernetesComputeDriver, OperatorNamespaceAllowlist, }; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] use openshell_driver_podman::{ComputeDriverService as PodmanDriverService, PodmanComputeDriver}; use prost::Message; use std::collections::HashMap; @@ -299,7 +299,7 @@ pub struct ManagedDriverProcess { } impl ManagedDriverProcess { - #[cfg(unix)] + #[cfg(all(unix, any(test, feature = "in-tree-compute-drivers")))] pub(crate) fn new(child: tokio::process::Child, socket_path: PathBuf) -> Self { Self { child: std::sync::Mutex::new(Some(child)), @@ -397,6 +397,7 @@ pub struct AcquiredRemoteDriverEndpoint { } impl AcquiredRemoteDriverEndpoint { + #[cfg(any(test, feature = "in-tree-compute-drivers"))] pub(crate) fn managed_builtin( driver_kind: ComputeDriverKind, channel: Channel, @@ -713,7 +714,7 @@ impl ComputeRuntime { self.lifecycle_gates.entry_count() } - #[cfg(not(target_os = "windows"))] + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] pub async fn new_docker( config: openshell_core::Config, docker_config: DockerComputeConfig, @@ -741,7 +742,7 @@ impl ComputeRuntime { .await } - #[cfg(not(target_os = "windows"))] + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] pub async fn new_kubernetes( config: KubernetesComputeConfig, store: Arc, @@ -792,7 +793,7 @@ impl ComputeRuntime { .await } - #[cfg(not(target_os = "windows"))] + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] pub async fn new_podman( config: PodmanComputeConfig, store: Arc, @@ -3334,20 +3335,34 @@ pub async fn connect_remote_compute_driver( name: impl Into, socket_path: &Path, ) -> Result { - let socket_path: PathBuf = socket_path.to_path_buf(); - let display_path = socket_path.clone(); - let channel = Endpoint::from_static("http://[::]:50051") - .connect_with_connector(service_fn(move |_: tonic::transport::Uri| { - let socket_path = socket_path.clone(); - async move { UnixStream::connect(socket_path).await.map(TokioIo::new) } - })) - .await - .map_err(|e| { - ComputeError::Message(format!( - "failed to connect to remote compute driver socket '{}': {e}", - display_path.display() - )) - })?; + let socket_path = socket_path.to_path_buf(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + let channel = loop { + let connector_path = socket_path.clone(); + match Endpoint::from_static("http://[::]:50051") + .connect_with_connector(service_fn(move |_: tonic::transport::Uri| { + let connector_path = connector_path.clone(); + async move { UnixStream::connect(connector_path).await.map(TokioIo::new) } + })) + .await + { + Ok(channel) => break channel, + Err(error) if tokio::time::Instant::now() < deadline => { + tracing::debug!( + socket = %socket_path.display(), + %error, + "waiting for remote compute driver socket" + ); + tokio::time::sleep(Duration::from_millis(250)).await; + } + Err(error) => { + return Err(ComputeError::Message(format!( + "failed to connect to remote compute driver socket '{}' within 30s: {error}", + socket_path.display() + ))); + } + } + }; Ok(AcquiredRemoteDriverEndpoint::unmanaged(name, channel)) } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 054771480..fe889d744 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -666,27 +666,11 @@ pub(crate) async fn run_server( // Pod lookups and TokenReview identity checks must match the sandbox // namespace and service account used by the Kubernetes driver. let kubernetes_config = - compute::driver_config::builtin::kubernetes_config_for_k8s_sa_bootstrap( - config_file.as_ref(), - )?; + compute::driver_config::kubernetes_sa_bootstrap_config(config_file.as_ref())?; let sandbox_namespace = kubernetes_config.namespace.clone(); let sandbox_service_account = kubernetes_config.service_account_name.clone(); - let namespace_validator = match kubernetes_config.workspace_mode { - openshell_driver_kubernetes::WorkspaceMode::Shared => { - auth::k8s_sa::NamespaceValidator::Exact(kubernetes_config.namespace) - } - openshell_driver_kubernetes::WorkspaceMode::Managed => { - auth::k8s_sa::NamespaceValidator::Prefix( - openshell_driver_kubernetes::managed_namespace_prefix( - &kubernetes_config.gateway_id, - ), - ) - } - openshell_driver_kubernetes::WorkspaceMode::Operator => { - let allowlist = operator_allowlist.clone().unwrap_or_default(); - auth::k8s_sa::NamespaceValidator::Allowlist(allowlist) - } - }; + let namespace_validator = + kubernetes_namespace_validator(&kubernetes_config, &operator_allowlist)?; match kube::Client::try_default().await { Ok(client) => { let resolver = Arc::new(auth::k8s_sa::LiveK8sResolver::new( @@ -1068,7 +1052,7 @@ async fn terminate_signal() { let _ = signal.recv().await; } -#[cfg(target_os = "windows")] +#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] fn unsupported_builtin_compute_driver(driver: ComputeDriverKind) -> compute::ComputeError { compute::ComputeError::Message(format!( "{} compute driver is unsupported on Windows", @@ -1076,9 +1060,54 @@ fn unsupported_builtin_compute_driver(driver: ComputeDriverKind) -> compute::Com )) } -type OperatorAllowlistArc = Option; +type OperatorAllowlistArc = Option; pub use compute::{DriverWatchStream, SharedComputeDriver}; +fn kubernetes_namespace_validator( + config: &compute::driver_config::KubernetesSaBootstrapConfig, + operator_allowlist: &OperatorAllowlistArc, +) -> Result { + match config.workspace_mode.as_str() { + "shared" => Ok(auth::k8s_sa::NamespaceValidator::Exact( + config.namespace.clone(), + )), + "managed" => Ok(auth::k8s_sa::NamespaceValidator::Prefix(format!( + "openshell-{}-", + config.gateway_id + ))), + "operator" => operator_allowlist + .clone() + .map(auth::k8s_sa::NamespaceValidator::Allowlist) + .ok_or_else(|| { + Error::config("Kubernetes operator namespace allowlist was not initialized") + }), + mode => Err(Error::config(format!( + "invalid Kubernetes workspace_mode '{mode}' for ServiceAccount bootstrap" + ))), + } +} + +fn validate_remote_compute_driver_config( + name: &str, + file: Option<&config_file::ConfigFile>, +) -> Result<()> { + if name != "kubernetes" + || !file.is_some_and(|file| file.openshell.drivers.contains_key("kubernetes")) + { + return Ok(()); + } + + let config = compute::driver_config::kubernetes_sa_bootstrap_config(file)?; + if config.workspace_mode == "operator" { + return Err(Error::config( + "Kubernetes workspace_mode 'operator' requires an in-process Kubernetes driver; \ + external Kubernetes compute drivers do not support operator mode", + )); + } + + Ok(()) +} + /// Opaque result returned by a compiled compute-driver factory. pub struct ComputeDriverBuildOutput { runtime: ComputeRuntime, @@ -1246,8 +1275,9 @@ impl ComputeDriverRegistry { /// Install every first-party compute driver linked into the standard gateway. #[must_use] pub fn install_default_compute_drivers() -> ComputeDriverRegistry { + #[allow(unused_mut)] let mut registry = ComputeDriverRegistry::new(); - #[cfg(not(target_os = "windows"))] + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] { registry .install( @@ -1289,7 +1319,7 @@ pub fn install_default_compute_drivers() -> ComputeDriverRegistry { ) .expect("unique vm registration"); } - #[cfg(target_os = "windows")] + #[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] for name in ["kubernetes", "podman", "docker", "vm"] { registry .install( @@ -1384,11 +1414,11 @@ impl ComputeDriverBuildContext<'_> { } } -#[cfg(target_os = "windows")] +#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] #[derive(Clone, Copy)] struct UnsupportedComputeDriverFactory; -#[cfg(target_os = "windows")] +#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl ComputeDriverFactory for UnsupportedComputeDriverFactory { async fn build( @@ -1407,11 +1437,11 @@ impl ComputeDriverFactory for UnsupportedComputeDriverFactory { } } -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[derive(Clone, Copy)] struct KubernetesComputeDriverFactory; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl ComputeDriverFactory for KubernetesComputeDriverFactory { async fn build( @@ -1440,11 +1470,11 @@ impl ComputeDriverFactory for KubernetesComputeDriverFactory { } } -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[derive(Clone, Copy)] struct DockerComputeDriverFactory; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl ComputeDriverFactory for DockerComputeDriverFactory { async fn build( @@ -1471,11 +1501,11 @@ impl ComputeDriverFactory for DockerComputeDriverFactory { } } -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[derive(Clone, Copy)] struct PodmanComputeDriverFactory; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl ComputeDriverFactory for PodmanComputeDriverFactory { async fn build( @@ -1501,11 +1531,11 @@ impl ComputeDriverFactory for PodmanComputeDriverFactory { } } -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[derive(Clone, Copy)] struct VmComputeDriverFactory; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl ComputeDriverFactory for VmComputeDriverFactory { async fn build( @@ -1569,6 +1599,7 @@ async fn build_compute_runtime( (output.runtime, output.operator_allowlist) } ConfiguredComputeDriver::Remote { name } => { + validate_remote_compute_driver_config(&name, driver_startup.file)?; let remote_config = compute::driver_config::remote_driver_config_from_context(driver_startup, &name)?; info!( @@ -1649,6 +1680,7 @@ fn resolve_configured_compute_driver( Ok(ConfiguredComputeDriver::Remote { name }) } +#[cfg(any(test, feature = "in-tree-compute-drivers"))] fn kubernetes_sandbox_jwt_expiry_disabled(config: &Config) -> bool { config .gateway_jwt @@ -1656,6 +1688,7 @@ fn kubernetes_sandbox_jwt_expiry_disabled(config: &Config) -> bool { .is_some_and(|jwt| jwt.ttl_secs == 0) } +#[cfg(feature = "in-tree-compute-drivers")] fn warn_if_kubernetes_sandbox_jwt_expiry_disabled(config: &Config) { if kubernetes_sandbox_jwt_expiry_disabled(config) { warn!( @@ -1731,6 +1764,7 @@ mod tests { allow_plaintext_service_http, bind_gateway_listeners, classify_initial_bytes, is_benign_tls_handshake_failure, kubernetes_sandbox_jwt_expiry_disabled, mint_gateway_extension_credential, serve_gateway_listener, + validate_remote_compute_driver_config, }; use openshell_core::{ ComputeDriverKind, Config, @@ -1767,6 +1801,28 @@ mod tests { ) } + #[test] + fn external_kubernetes_operator_workspace_mode_is_rejected() { + let file: crate::config_file::ConfigFile = toml::from_str( + r#" +[openshell.drivers.kubernetes] +socket_path = "/run/openshell/kubernetes.sock" +workspace_mode = "operator" +operator_namespace_label = "openshell.ai/workspace=true" +"#, + ) + .expect("valid config"); + + let error = validate_remote_compute_driver_config("kubernetes", Some(&file)) + .expect_err("external operator mode must fail closed"); + + assert!( + error + .to_string() + .contains("external Kubernetes compute drivers do not support operator mode") + ); + } + #[test] fn plaintext_extension_endpoint_is_rejected_unless_explicitly_opted_out() { let issuer = extension_test_issuer(); diff --git a/crates/openshell-server/src/otel_tracing.rs b/crates/openshell-server/src/otel_tracing.rs index d29f3b77f..d8fb17a62 100644 --- a/crates/openshell-server/src/otel_tracing.rs +++ b/crates/openshell-server/src/otel_tracing.rs @@ -29,6 +29,12 @@ use opentelemetry_sdk::trace::SdkTracerProvider; use tracing::Subscriber; use tracing_subscriber::registry::LookupSpan; +#[cfg(feature = "in-tree-compute-drivers")] +const COMPUTE_DRIVER_TARGET_PREFIX: &str = + openshell_driver_podman::otel_tracing::IN_PROCESS_TARGET_PREFIX; +#[cfg(not(feature = "in-tree-compute-drivers"))] +const COMPUTE_DRIVER_TARGET_PREFIX: &str = "\0"; + use crate::config_file::OtlpConfig; /// `service.name` reported when the config file does not override it. @@ -97,7 +103,7 @@ where openshell_otel::layer_excluding_target_prefix( provider, INSTRUMENTATION_SCOPE, - openshell_driver_podman::otel_tracing::IN_PROCESS_TARGET_PREFIX, + COMPUTE_DRIVER_TARGET_PREFIX, ) } diff --git a/crates/openshell-server/src/tracing_setup.rs b/crates/openshell-server/src/tracing_setup.rs index 41b4e619c..aff5c05ec 100644 --- a/crates/openshell-server/src/tracing_setup.rs +++ b/crates/openshell-server/src/tracing_setup.rs @@ -51,23 +51,29 @@ pub fn install( enable_podman_export: bool, ) -> (TracingHandle, Option) { let (tracer_provider, setup_error) = crate::otel_tracing::provider_for(otlp_config); + #[cfg(feature = "in-tree-compute-drivers")] let podman_endpoint = enable_podman_export .then_some(otlp_config) .flatten() .map(|config| config.endpoint.as_str()); + #[cfg(feature = "in-tree-compute-drivers")] let (podman_tracer_provider, podman_setup_error) = openshell_driver_podman::otel_tracing::provider_for(podman_endpoint); + #[cfg(not(feature = "in-tree-compute-drivers"))] + let (podman_tracer_provider, podman_setup_error): ( + Option, + Option, + ) = { + let _ = enable_podman_export; + (None, None) + }; tracing_subscriber::registry() .with(env_filter) .with(tracing_subscriber::fmt::layer()) .with(tracing_log_bus.layer()) .with(tracer_provider.as_ref().map(crate::otel_tracing::layer)) - .with( - podman_tracer_provider - .as_ref() - .map(openshell_driver_podman::otel_tracing::in_process_layer), - ) + .with(podman_in_process_layer(&podman_tracer_provider)) .init(); ( @@ -79,6 +85,28 @@ pub fn install( ) } +#[cfg(feature = "in-tree-compute-drivers")] +fn podman_in_process_layer( + provider: &Option, +) -> Option> +where + S: tracing::Subscriber + for<'span> tracing_subscriber::registry::LookupSpan<'span>, +{ + provider + .as_ref() + .map(openshell_driver_podman::otel_tracing::in_process_layer) +} + +#[cfg(not(feature = "in-tree-compute-drivers"))] +fn podman_in_process_layer( + _provider: &Option, +) -> Option> +where + S: tracing::Subscriber + for<'span> tracing_subscriber::registry::LookupSpan<'span>, +{ + None +} + #[cfg(test)] mod tests { use super::*; diff --git a/e2e/docker/Dockerfile.external-kubernetes-gateway b/e2e/docker/Dockerfile.external-kubernetes-gateway new file mode 100644 index 000000000..5d650bae8 --- /dev/null +++ b/e2e/docker/Dockerfile.external-kubernetes-gateway @@ -0,0 +1,30 @@ +# syntax=docker/dockerfile:1.4 + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +ARG GATEWAY_BASE_IMAGE=gcr.io/distroless/cc-debian13:nonroot@sha256:d97bc0a941b8d4be647dc0ee75b264ddbb772f1ac5ba690a4309c00723b23775 +FROM ${GATEWAY_BASE_IMAGE} + +ARG TARGETARCH +ARG SUPERVISOR_IMAGE=ghcr.io/nvidia/openshell/supervisor:latest +COPY deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-gateway /usr/local/bin/openshell-gateway +COPY deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-driver-kubernetes /usr/local/bin/openshell-driver-kubernetes + +ENV OPENSHELL_DRIVERS=kubernetes \ + OPENSHELL_COMPUTE_DRIVER_SOCKET=/var/run/openshell-compute/driver/driver.sock \ + OPENSHELL_GATEWAY_ID=openshell \ + OPENSHELL_SANDBOX_NAMESPACE=openshell \ + OPENSHELL_K8S_SANDBOX_SERVICE_ACCOUNT=openshell-sandbox \ + OPENSHELL_SANDBOX_IMAGE=ghcr.io/nvidia/openshell-community/sandboxes/base:latest \ + OPENSHELL_SANDBOX_IMAGE_PULL_POLICY=IfNotPresent \ + OPENSHELL_GRPC_ENDPOINT=http://openshell.openshell.svc.cluster.local:8080 \ + OPENSHELL_SUPERVISOR_IMAGE=${SUPERVISOR_IMAGE} \ + OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY=IfNotPresent \ + OPENSHELL_SUPERVISOR_SIDELOAD_METHOD=init-container \ + OPENSHELL_K8S_TOPOLOGY=combined + +USER 1000:1000 +EXPOSE 8080 +ENTRYPOINT ["/usr/local/bin/openshell-gateway"] +CMD ["--bind-address", "0.0.0.0", "--port", "8080"] diff --git a/e2e/helm-plugins/openshell-external-compute-driver/kustomization.yaml b/e2e/helm-plugins/openshell-external-compute-driver/kustomization.yaml new file mode 100644 index 000000000..3c01ab696 --- /dev/null +++ b/e2e/helm-plugins/openshell-external-compute-driver/kustomization.yaml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - rendered.yaml +patches: + - path: workload-patch.yaml + target: + group: apps + version: v1 + kind: StatefulSet + name: openshell +replacements: + - source: + group: apps + version: v1 + kind: StatefulSet + name: openshell + fieldPath: spec.template.spec.containers.[name=openshell-gateway].image + targets: + - select: + group: apps + version: v1 + kind: StatefulSet + name: openshell + fieldPaths: + - spec.template.spec.containers.[name=kubernetes-compute-driver].image diff --git a/e2e/helm-plugins/openshell-external-compute-driver/plugin.yaml b/e2e/helm-plugins/openshell-external-compute-driver/plugin.yaml new file mode 100644 index 000000000..2a58ba4f0 --- /dev/null +++ b/e2e/helm-plugins/openshell-external-compute-driver/plugin.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v1 +type: postrenderer/v1 +name: openshell-external-compute-driver +version: 0.1.0 +runtime: subprocess +runtimeConfig: + platformCommand: + - command: ${HELM_PLUGIN_DIR}/post-renderer.sh diff --git a/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh b/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh new file mode 100755 index 000000000..4d9d2eeff --- /dev/null +++ b/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Helm post-renderer for the external Kubernetes compute-driver smoke test. +# It keeps the test-only sidecar and Unix socket plumbing out of the chart. + +set -euo pipefail + +plugin_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +work_dir="$(mktemp -d "${TMPDIR:-/tmp}/openshell-external-compute-driver.XXXXXX")" +trap 'rm -rf "${work_dir}"' EXIT + +cp "${plugin_dir}/kustomization.yaml" "${work_dir}/kustomization.yaml" +cp "${plugin_dir}/workload-patch.yaml" "${work_dir}/workload-patch.yaml" +tee "${work_dir}/rendered.yaml" >/dev/null + +kubectl kustomize "${work_dir}" diff --git a/e2e/helm-plugins/openshell-external-compute-driver/workload-patch.yaml b/e2e/helm-plugins/openshell-external-compute-driver/workload-patch.yaml new file mode 100644 index 000000000..2b041e864 --- /dev/null +++ b/e2e/helm-plugins/openshell-external-compute-driver/workload-patch.yaml @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: openshell +spec: + template: + spec: + containers: + - name: openshell-gateway + volumeMounts: + - name: compute-driver-socket + mountPath: /var/run/openshell-compute + - name: kubernetes-compute-driver + image: replaced-by-kustomize + imagePullPolicy: IfNotPresent + command: + - /usr/local/bin/openshell-driver-kubernetes + args: + - --bind-socket + - /var/run/openshell-compute/driver/driver.sock + securityContext: + runAsNonRoot: true + runAsUser: 1000 + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: compute-driver-socket + mountPath: /var/run/openshell-compute + resources: {} + volumes: + - name: compute-driver-socket + emptyDir: {} diff --git a/e2e/no-compute-driver-gateway.sh b/e2e/no-compute-driver-gateway.sh new file mode 100755 index 000000000..7ad3896ca --- /dev/null +++ b/e2e/no-compute-driver-gateway.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${ROOT}" + +echo "Building gateway without compiled compute drivers..." +cargo build -p openshell-server --bin openshell-gateway \ + --no-default-features --features telemetry + +dependency_tree="$(cargo tree -p openshell-server \ + --no-default-features --features telemetry --edges normal)" +for driver in \ + openshell-driver-docker \ + openshell-driver-kubernetes \ + openshell-driver-podman \ + openshell-driver-vm; do + if grep -q "${driver} v" <<<"${dependency_tree}"; then + echo "ERROR: driver-free gateway dependency graph contains ${driver}" >&2 + exit 1 + fi +done + +"${ROOT}/target/debug/openshell-gateway" --version +echo "Driver-free gateway build passed." diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 26671ccb8..1960f8358 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -53,6 +53,7 @@ DRIVER_BIN="${OPENSHELL_VM_DRIVER_BIN:-${ROOT}/target/debug/openshell-driver-vm} CLI_BIN="${OPENSHELL_BIN:-${ROOT}/target/debug/openshell}" E2E_TEST_OVERRIDE="${OPENSHELL_E2E_VM_TEST:-}" E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES:-e2e-vm}" +SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-${COMMUNITY_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}}" # The VM driver places `compute-driver.sock` under `[openshell.drivers.vm].state_dir`. # AF_UNIX SUN_LEN is 104 bytes on macOS (108 on Linux), so paths anchored @@ -97,7 +98,14 @@ fi build_packages=() if [ -z "${OPENSHELL_GATEWAY_BIN:-}" ]; then - build_packages+=(-p openshell-server) + if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + echo "==> Building driver-free openshell-gateway" + cargo build \ + -p openshell-server --bin openshell-gateway \ + --no-default-features --features telemetry + else + build_packages+=(-p openshell-server) + fi else echo "==> Using prebuilt openshell-gateway at ${GATEWAY_BIN}" fi @@ -165,6 +173,9 @@ GATEWAY_DB="${RUN_STATE_DIR}/gateway.db" JWT_DIR="${RUN_STATE_DIR}/jwt" PKI_DIR="${RUN_STATE_DIR}/pki" GATEWAY_NAME="openshell-e2e-vm-${HOST_PORT}" +DRIVER_PID="" +DRIVER_LOG="${RUN_STATE_DIR}/vm-driver.log" +DRIVER_SOCKET="${RUN_STATE_DIR}/compute-driver.sock" # ── Cleanup (trap) ─────────────────────────────────────────────────── @@ -188,6 +199,7 @@ cleanup() { kill -KILL "${gateway_pid}" 2>/dev/null || true wait "${gateway_pid}" 2>/dev/null || true fi + e2e_stop_process "${DRIVER_PID}" "external VM compute driver" # On failure, keep the VM console log for debugging. We deliberately # print it instead of leaving it on disk because the state dir gets @@ -196,6 +208,11 @@ cleanup() { echo "=== gateway log (preserved for debugging) ===" cat "${GATEWAY_LOG}" 2>/dev/null || true echo "=== end gateway log ===" + if [ -f "${DRIVER_LOG}" ]; then + echo "=== external VM compute driver log ===" + cat "${DRIVER_LOG}" 2>/dev/null || true + echo "=== end external VM compute driver log ===" + fi local console while IFS= read -r -d '' console; do @@ -261,6 +278,11 @@ gateway_id = "${GATEWAY_NAME}" ttl_secs = 0 [openshell.drivers.vm] +EOF +if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + printf 'socket_path = "%s"\n' "${DRIVER_SOCKET}" >>"${GATEWAY_CONFIG}" +else + cat >>"${GATEWAY_CONFIG}" <"${DRIVER_LOG}" 2>&1 & + DRIVER_PID=$! + e2e_wait_for_socket \ + "${DRIVER_SOCKET}" "${DRIVER_PID}" "external VM compute driver" 60 +fi GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" diff --git a/e2e/support/gateway-common.sh b/e2e/support/gateway-common.sh index a3eef4bd1..1b2b8e414 100644 --- a/e2e/support/gateway-common.sh +++ b/e2e/support/gateway-common.sh @@ -217,8 +217,14 @@ e2e_build_gateway_binaries() { if [ -z "${OPENSHELL_GATEWAY_BIN:-}" ]; then echo "Building openshell-gateway..." - cargo build "${jobs[@]}" \ - -p openshell-server --bin openshell-gateway + if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + cargo build "${jobs[@]}" \ + -p openshell-server --bin openshell-gateway \ + --no-default-features --features telemetry + else + cargo build "${jobs[@]}" \ + -p openshell-server --bin openshell-gateway + fi else echo "Using prebuilt openshell gateway at ${OPENSHELL_GATEWAY_BIN}" fi @@ -241,6 +247,59 @@ e2e_build_gateway_binaries() { fi } +e2e_build_external_driver() { + local root=$1 + local package=$2 + local binary=$3 + local output_var=$4 + local target_dir + local jobs=() + + if [ -n "${CARGO_BUILD_JOBS:-}" ]; then + jobs=(-j "${CARGO_BUILD_JOBS}") + fi + target_dir="$(e2e_cargo_target_dir "${root}")" + printf -v "${output_var}" '%s' "${target_dir}/debug/${binary}" + echo "Building external ${binary}..." + cargo build "${jobs[@]}" -p "${package}" --bin "${binary}" + if [ ! -x "${!output_var}" ]; then + echo "ERROR: expected external driver binary at ${!output_var}" >&2 + exit 1 + fi +} + +e2e_wait_for_socket() { + local socket_path=$1 + local process_pid=$2 + local process_label=$3 + local timeout="${4:-30}" + local elapsed=0 + + while [ "${elapsed}" -lt "${timeout}" ]; do + if [ -S "${socket_path}" ]; then + return 0 + fi + if ! kill -0 "${process_pid}" 2>/dev/null; then + echo "ERROR: ${process_label} exited before creating ${socket_path}" >&2 + return 1 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + echo "ERROR: ${process_label} did not create ${socket_path} within ${timeout}s" >&2 + return 1 +} + +e2e_stop_process() { + local process_pid=$1 + local process_label=$2 + if [ -n "${process_pid}" ] && kill -0 "${process_pid}" 2>/dev/null; then + echo "Stopping ${process_label} (pid ${process_pid})..." + kill "${process_pid}" 2>/dev/null || true + wait "${process_pid}" 2>/dev/null || true + fi +} + e2e_write_gateway_args_file() { local args_file=$1 shift diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index 15e9d3466..958ea1b0e 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -109,6 +109,11 @@ GATEWAY_PID="" GATEWAY_LOG="${WORKDIR}/gateway.log" GATEWAY_PID_FILE="${WORKDIR}/gateway.pid" GATEWAY_ARGS_FILE="${WORKDIR}/gateway.args" +DRIVER_BIN="" +DRIVER_PID="" +DRIVER_LOG="${WORKDIR}/docker-driver.log" +DRIVER_SOCKET="${WORKDIR}/compute-driver.sock" +DRIVER_CONFIG="${WORKDIR}/docker-driver.toml" E2E_NAMESPACE="" DOCKER_NETWORK_NAME="" DOCKER_NETWORK_CONNECTED_CONTAINER="" @@ -134,6 +139,7 @@ cleanup() { local exit_code=$? e2e_stop_gateway "${GATEWAY_PID}" "${GATEWAY_PID_FILE}" + e2e_stop_process "${DRIVER_PID}" "external Docker compute driver" if [ "${exit_code}" -ne 0 ] \ && [ -n "${E2E_NAMESPACE}" ] \ @@ -182,6 +188,11 @@ cleanup() { fi e2e_print_gateway_log_on_failure "${exit_code}" "${GATEWAY_LOG}" + if [ "${exit_code}" -ne 0 ] && [ -f "${DRIVER_LOG}" ]; then + echo "=== external Docker compute driver log ===" + cat "${DRIVER_LOG}" || true + echo "=== end external Docker compute driver log ===" + fi rm -rf "${WORKDIR}" 2>/dev/null || true } @@ -426,6 +437,10 @@ ensure_sandbox_image_available() { } e2e_build_gateway_binaries "${ROOT}" TARGET_DIR GATEWAY_BIN CLI_BIN +if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + e2e_build_external_driver \ + "${ROOT}" openshell-driver-docker openshell-driver-docker DRIVER_BIN +fi SUPERVISOR_IMAGE="$(resolve_docker_supervisor_image)" build_local_docker_supervisor_image_if_required "${SUPERVISOR_IMAGE}" @@ -495,21 +510,51 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" fi fi printf '[openshell.drivers.docker]\n' - printf 'sandbox_namespace = %s\n' "$(toml_string "${E2E_NAMESPACE}")" - printf 'network_name = %s\n' "$(toml_string "${DOCKER_NETWORK_NAME}")" - printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" - printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" - printf 'image_pull_policy = %s\n' "$(toml_string "${SANDBOX_IMAGE_PULL_POLICY}")" - printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" - printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" - printf 'guest_tls_key = %s\n' "$(toml_string "${PKI_DIR}/client/tls.key")" - printf 'enable_bind_mounts = true\n' - printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" - if [ -n "${GATEWAY_HOST_ALIAS_IP}" ]; then - printf 'host_gateway_ip = %s\n' "$(toml_string "${GATEWAY_HOST_ALIAS_IP}")" + if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")" + else + printf 'sandbox_namespace = %s\n' "$(toml_string "${E2E_NAMESPACE}")" + printf 'network_name = %s\n' "$(toml_string "${DOCKER_NETWORK_NAME}")" + printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" + printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" + printf 'image_pull_policy = %s\n' "$(toml_string "${SANDBOX_IMAGE_PULL_POLICY}")" + printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" + printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" + printf 'guest_tls_key = %s\n' "$(toml_string "${PKI_DIR}/client/tls.key")" + printf 'enable_bind_mounts = true\n' + printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" + if [ -n "${GATEWAY_HOST_ALIAS_IP}" ]; then + printf 'host_gateway_ip = %s\n' "$(toml_string "${GATEWAY_HOST_ALIAS_IP}")" + fi fi } > "${GATEWAY_CONFIG}" +if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + { + printf 'sandbox_namespace = %s\n' "$(toml_string "${E2E_NAMESPACE}")" + printf 'network_name = %s\n' "$(toml_string "${DOCKER_NETWORK_NAME}")" + printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" + printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" + printf 'image_pull_policy = %s\n' "$(toml_string "${SANDBOX_IMAGE_PULL_POLICY}")" + printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" + printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" + printf 'guest_tls_key = %s\n' "$(toml_string "${PKI_DIR}/client/tls.key")" + printf 'enable_bind_mounts = true\n' + printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" + if [ -n "${GATEWAY_HOST_ALIAS_IP}" ]; then + printf 'host_gateway_ip = %s\n' "$(toml_string "${GATEWAY_HOST_ALIAS_IP}")" + fi + } >"${DRIVER_CONFIG}" + "${DRIVER_BIN}" \ + --bind-socket "${DRIVER_SOCKET}" \ + --config "${DRIVER_CONFIG}" \ + --gateway-bind "127.0.0.1:${HOST_PORT}" \ + >"${DRIVER_LOG}" 2>&1 & + DRIVER_PID=$! + e2e_wait_for_socket \ + "${DRIVER_SOCKET}" "${DRIVER_PID}" "external Docker compute driver" +fi + GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" --port "${HOST_PORT}" diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index f6d0efc4e..b6ebb7ca0 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -380,6 +380,7 @@ run_scenario() { --set "image.tag=${IMAGE_TAG_VALUE}" \ --set "supervisor.image.repository=${REGISTRY_VALUE}/supervisor" \ --set "supervisor.image.tag=${IMAGE_TAG_VALUE}" \ + "${helm_post_renderer_args[@]}" \ "$@" \ --wait --timeout 5m HELM_INSTALLED=1 @@ -554,6 +555,7 @@ if [ -z "${OPENSHELL_E2E_KUBE_BUILD_IMAGES+x}" ]; then fi fi +reuse_supervisor_image=0 if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then REGISTRY_VALUE="${OPENSHELL_REGISTRY:-openshell}" IMAGE_TAG_VALUE="${IMAGE_TAG:-e2e-${CLUSTER_NAME:-local}}" @@ -655,10 +657,43 @@ fi if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then require_cmd docker echo "Building local Kubernetes e2e images (${REGISTRY_VALUE}/{gateway,supervisor}:${IMAGE_TAG_VALUE})..." - CONTAINER_ENGINE=docker IMAGE_REGISTRY="${REGISTRY_VALUE}" IMAGE_TAG="${IMAGE_TAG_VALUE}" \ - bash "${ROOT}/tasks/scripts/docker-build-image.sh" gateway - CONTAINER_ENGINE=docker IMAGE_REGISTRY="${REGISTRY_VALUE}" IMAGE_TAG="${IMAGE_TAG_VALUE}" \ - bash "${ROOT}/tasks/scripts/docker-build-image.sh" supervisor + if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + if [ "$(uname -s)" != "Linux" ]; then + echo "ERROR: external Kubernetes driver image composition currently requires a Linux build host." >&2 + exit 2 + fi + cargo build -p openshell-server --bin openshell-gateway \ + --no-default-features --features telemetry,bundled-z3 + cargo build -p openshell-driver-kubernetes --bin openshell-driver-kubernetes + case "$(uname -m)" in + x86_64) external_arch=amd64 ;; + aarch64|arm64) external_arch=arm64 ;; + *) echo "ERROR: unsupported external Kubernetes driver architecture: $(uname -m)" >&2; exit 2 ;; + esac + external_stage="${ROOT}/deploy/docker/.build/prebuilt-binaries/${external_arch}" + mkdir -p "${external_stage}" + cp "${ROOT}/target/debug/openshell-gateway" "${external_stage}/openshell-gateway" + cp "${ROOT}/target/debug/openshell-driver-kubernetes" \ + "${external_stage}/openshell-driver-kubernetes" + docker build \ + --build-arg "TARGETARCH=${external_arch}" \ + --build-arg "SUPERVISOR_IMAGE=${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}" \ + --tag "${REGISTRY_VALUE}/gateway:${IMAGE_TAG_VALUE}" \ + --file "${ROOT}/e2e/docker/Dockerfile.external-kubernetes-gateway" \ + "${ROOT}" + else + CONTAINER_ENGINE=docker IMAGE_REGISTRY="${REGISTRY_VALUE}" IMAGE_TAG="${IMAGE_TAG_VALUE}" \ + bash "${ROOT}/tasks/scripts/docker-build-image.sh" gateway + fi + supervisor_image="${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}" + if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" != "1" ] \ + || ! docker image inspect "${supervisor_image}" >/dev/null 2>&1; then + CONTAINER_ENGINE=docker IMAGE_REGISTRY="${REGISTRY_VALUE}" IMAGE_TAG="${IMAGE_TAG_VALUE}" \ + bash "${ROOT}/tasks/scripts/docker-build-image.sh" supervisor + else + reuse_supervisor_image=1 + echo "Reusing existing supervisor image ${supervisor_image}" + fi fi if [ -n "${import_cluster_name}" ]; then @@ -671,6 +706,20 @@ if [ -n "${import_cluster_name}" ]; then --mode direct >/dev/null fi done +elif [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ] \ + && [[ "${KUBE_CONTEXT}" == kind-* ]] \ + && command -v kind >/dev/null 2>&1; then + kind_cluster_name="${KUBE_CONTEXT#kind-}" + kind_images=("${REGISTRY_VALUE}/gateway:${IMAGE_TAG_VALUE}") + # The CI workflow loads its published supervisor archive before invoking this + # wrapper. Only load a supervisor image here when this script rebuilt it. + if [ "${reuse_supervisor_image}" != "1" ]; then + kind_images+=("${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}") + fi + for image in "${kind_images[@]}"; do + echo "Loading ${image} into kind cluster ${kind_cluster_name}..." + kind load docker-image "${image}" --name "${kind_cluster_name}" + done fi # The Kubernetes compute driver creates and watches Sandbox CRs reconciled @@ -689,7 +738,18 @@ if [ "${OPENSHELL_E2E_CREDENTIAL_DRIVERS:-0}" = "1" ] \ fi helm_extra_args=() +helm_post_renderer_args=() helm_extra_args+=(--set "server.telemetryEnabled=${OPENSHELL_TELEMETRY_ENABLED}") +if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" != "1" ]; then + echo "ERROR: external Kubernetes driver e2e requires OPENSHELL_E2E_KUBE_BUILD_IMAGES=1." >&2 + exit 2 + fi + export HELM_PLUGINS="${ROOT}/e2e/helm-plugins" + helm_post_renderer_args+=( + --post-renderer openshell-external-compute-driver + ) +fi if [ -n "${HOST_GATEWAY_IP}" ]; then helm_extra_args+=(--set "server.hostGatewayIP=${HOST_GATEWAY_IP}") fi @@ -811,6 +871,7 @@ else --set "supervisor.image.repository=${REGISTRY_VALUE}/supervisor" \ --set "supervisor.image.tag=${IMAGE_TAG_VALUE}" \ "${helm_extra_args[@]}" \ + "${helm_post_renderer_args[@]}" \ --wait --timeout 5m HELM_INSTALLED=1 diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index cd52e007a..bb22ee737 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -92,6 +92,10 @@ GATEWAY_PID="" GATEWAY_LOG="${WORKDIR}/gateway.log" GATEWAY_PID_FILE="${WORKDIR}/gateway.pid" GATEWAY_ARGS_FILE="${WORKDIR}/gateway.args" +DRIVER_BIN="" +DRIVER_PID="" +DRIVER_LOG="${WORKDIR}/podman-driver.log" +DRIVER_SOCKET="${WORKDIR}/compute-driver.sock" E2E_NAMESPACE="" PODMAN_NETWORK_NAME="" PODMAN_NETWORK_MANAGED=0 @@ -114,6 +118,7 @@ cleanup() { local exit_code=$? e2e_stop_gateway "${GATEWAY_PID}" "${GATEWAY_PID_FILE}" + e2e_stop_process "${DRIVER_PID}" "external Podman compute driver" local sandbox_ids="" if command -v podman >/dev/null 2>&1; then @@ -159,6 +164,11 @@ cleanup() { fi e2e_print_gateway_log_on_failure "${exit_code}" "${GATEWAY_LOG}" + if [ "${exit_code}" -ne 0 ] && [ -f "${DRIVER_LOG}" ]; then + echo "=== external Podman compute driver log ===" + cat "${DRIVER_LOG}" || true + echo "=== end external Podman compute driver log ===" + fi if [ "${exit_code}" -ne 0 ] && [ -f "${PODMAN_SERVICE_LOG}" ]; then echo "=== podman service log (preserved for debugging) ===" cat "${PODMAN_SERVICE_LOG}" || true @@ -363,6 +373,10 @@ fi ensure_podman_api_socket e2e_build_gateway_binaries "${ROOT}" TARGET_DIR GATEWAY_BIN CLI_BIN +if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + e2e_build_external_driver \ + "${ROOT}" openshell-driver-podman openshell-driver-podman DRIVER_BIN +fi SUPERVISOR_IMAGE="$(resolve_podman_supervisor_image)" ensure_podman_supervisor_image "${SUPERVISOR_IMAGE}" @@ -443,6 +457,9 @@ cp "${ROOT}/deploy/rpm/gateway.toml.default" "${GATEWAY_CONFIG}" fi fi printf '\n[openshell.drivers.podman]\n' + if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")" + else # The Podman driver scopes isolation by network rather than namespace. printf 'network_name = %s\n' "$(toml_string "${PODMAN_NETWORK_NAME}")" printf 'gateway_port = %s\n' "${HOST_PORT}" @@ -464,8 +481,28 @@ cp "${ROOT}/deploy/rpm/gateway.toml.default" "${GATEWAY_CONFIG}" if [ -n "${OPENSHELL_PODMAN_SOCKET:-}" ]; then printf 'socket_path = %s\n' "$(toml_string "${OPENSHELL_PODMAN_SOCKET}")" fi + fi } >> "${GATEWAY_CONFIG}" +if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + OPENSHELL_COMPUTE_DRIVER_SOCKET="${DRIVER_SOCKET}" \ + OPENSHELL_PODMAN_SOCKET="${OPENSHELL_PODMAN_SOCKET:-}" \ + OPENSHELL_SANDBOX_IMAGE="${SANDBOX_IMAGE}" \ + OPENSHELL_SANDBOX_IMAGE_PULL_POLICY="missing" \ + OPENSHELL_GATEWAY_PORT="${HOST_PORT}" \ + OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ + OPENSHELL_STOP_TIMEOUT="${PODMAN_STOP_TIMEOUT_SECS}" \ + OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_IMAGE}" \ + OPENSHELL_PODMAN_TLS_CA="${PKI_DIR}/ca.crt" \ + OPENSHELL_PODMAN_TLS_CERT="${PKI_DIR}/client/tls.crt" \ + OPENSHELL_PODMAN_TLS_KEY="${PKI_DIR}/client/tls.key" \ + OPENSHELL_ENABLE_BIND_MOUNTS=true \ + "${DRIVER_BIN}" >"${DRIVER_LOG}" 2>&1 & + DRIVER_PID=$! + e2e_wait_for_socket \ + "${DRIVER_SOCKET}" "${DRIVER_PID}" "external Podman compute driver" +fi + GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" # compute_drivers comes from the RPM template. Override the loopback address diff --git a/examples/governance-interceptor/Cargo.lock b/examples/governance-interceptor/Cargo.lock index 97f97aecc..1f23482a7 100644 --- a/examples/governance-interceptor/Cargo.lock +++ b/examples/governance-interceptor/Cargo.lock @@ -998,6 +998,7 @@ dependencies = [ "prost", "prost-types", "protoc-bin-vendored", + "rustix", "serde", "serde_json", "thiserror", diff --git a/tasks/test.toml b/tasks/test.toml index bef8baf2b..9f47d0dcd 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -184,6 +184,30 @@ run = "e2e/rust/e2e-kubernetes.sh" description = "Start openshell-gateway with the VM compute driver and run VM e2e tests" run = "e2e/rust/e2e-vm.sh" +["e2e:gateway:no-compute-drivers"] +description = "Build and launch-check openshell-gateway without compiled compute drivers" +run = "bash e2e/no-compute-driver-gateway.sh" + +["e2e:docker:external-driver"] +description = "Run Docker smoke E2E with a driver-free gateway and external Docker driver binary" +env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1" } +run = "e2e/rust/e2e-docker.sh" + +["e2e:podman:external-driver"] +description = "Run Podman E2E with a driver-free gateway and external Podman driver binary" +env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_PODMAN_TEST = "smoke" } +run = "e2e/rust/e2e-podman.sh" + +["e2e:vm:external-driver"] +description = "Run VM E2E with a driver-free gateway and external VM driver binary" +env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1" } +run = "e2e/rust/e2e-vm.sh" + +["e2e:kubernetes:external-driver"] +description = "Run Kubernetes smoke E2E with a driver-free gateway and external Kubernetes driver sidecar" +env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_KUBE_BUILD_IMAGES = "1", OPENSHELL_E2E_KUBE_TEST = "smoke" } +run = "e2e/rust/e2e-kubernetes.sh" + ["e2e:docker"] description = "Run smoke e2e against a standalone gateway with the Docker compute driver" run = "e2e/rust/e2e-docker.sh"