diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 7da42ec94c..afd9cc2f29 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -19,6 +19,13 @@ The target deployment flow is: 4. The CLI registers a reachable gateway endpoint with `openshell gateway add`. 5. The gateway creates sandboxes through the selected compute driver. +The standard gateway binary explicitly installs its compiled Docker, Podman, +Kubernetes, and VM registrations at startup. With no configured driver, the +gateway probes only installed registrations in priority order (Kubernetes, +Podman, then Docker); VM has no probe and remains opt-in. A custom gateway +binary may install a different set, so confirm the binary's registered drivers +when auto-detection reports that no suitable driver is available. + For local evaluation only, TLS may be disabled and the gateway can be reached through `http://127.0.0.1:`. ## Prerequisites diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 7373c72a62..ae94b6bd96 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -91,6 +91,28 @@ The gateway records driver identity and version from the startup capability response. Elevated gateway info reports that initialized driver snapshot instead of re-querying drivers on each request. +## Compiled Driver Selection + +The gateway binary explicitly installs the compute drivers compiled into that +binary before entering server startup. The server selects a configured driver +by normalized registry name. When no driver is configured, it evaluates only +the installed drivers' probes in registered priority order, records every +available registration, and selects the first. Drivers without a probe, +including VM, remain opt-in. + +Startup computes this selection once after merging configuration. The same +selection drives authentication defaults and runtime construction, so a probe +result cannot change which driver is constructed later in startup. + +This follows the same composition model as SQLx's `Any` drivers: the binary +defines the available implementation set, while the runtime consumes a generic +registry. Adding or removing a compiled driver therefore changes registration +rather than the server's selection flow. Alternate gateway binaries can install +their own `ComputeDriverFactory` registrations and hand the completed registry +to `run_cli_with_compute_drivers`; factories receive merged driver config and +finish through the same in-process runtime adapter. A configured UDS endpoint +still takes precedence over a compiled registration with the same name. + ## Stop and Start Lifecycle The gateway persists lifecycle intent before mutating compute: diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index fcbdeb73b6..62411e20b5 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -208,7 +208,9 @@ pub fn detect_driver() -> Option { None } -fn is_podman_available() -> bool { +/// Return whether a responsive local Podman API socket is available. +#[must_use] +pub fn is_podman_available() -> bool { detect_podman_socket().is_some() } @@ -266,7 +268,9 @@ fn podman_socket_candidates_from_env( candidates } -fn is_docker_available() -> bool { +/// Return whether a responsive local Docker API socket is available. +#[must_use] +pub fn is_docker_available() -> bool { detect_docker_socket().is_some() } diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 0a03ca1448..4a1df2a63b 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -18,8 +18,8 @@ use crate::compute::driver_config::GuestTlsPaths; use crate::config_file::{self, ConfigFile, GatewayFileSection}; use crate::defaults::{self, LocalTlsPaths}; use crate::{ - ServerStartupConfig, configured_compute_driver_for_startup, run_server, - tracing_bus::TracingLogBus, + ComputeDriverRegistry, ServerStartupConfig, configured_compute_driver_for_startup, + install_default_compute_drivers, run_server, tracing_bus::TracingLogBus, }; /// `OpenShell` gateway process - gRPC and HTTP server with protocol multiplexing. @@ -223,6 +223,11 @@ pub fn command() -> Command { } pub async fn run_cli() -> Result<()> { + run_cli_with_compute_drivers(install_default_compute_drivers()).await +} + +/// Run the gateway CLI with the compute drivers linked by the binary. +pub async fn run_cli_with_compute_drivers(compute_drivers: ComputeDriverRegistry) -> Result<()> { rustls::crypto::ring::default_provider() .install_default() .map_err(|e| miette::miette!("failed to install rustls crypto provider: {e:?}"))?; @@ -232,11 +237,15 @@ pub async fn run_cli() -> Result<()> { match cli.command { Some(Commands::GenerateCerts(args)) => certgen::run(args).await, - None => Box::pin(run_from_args(cli.run, matches)).await, + None => Box::pin(run_from_args(cli.run, matches, compute_drivers)).await, } } -fn prepare_server_config(args: &mut RunArgs, matches: &ArgMatches) -> Result { +fn prepare_server_config( + args: &mut RunArgs, + matches: &ArgMatches, + compute_drivers: &ComputeDriverRegistry, +) -> Result { // Load TOML when explicitly requested, or from the default XDG location // when that file exists. Missing default config is not an error: runtime // defaults and OPENSHELL_* env vars are enough for package-managed starts. @@ -250,6 +259,10 @@ fn prepare_server_config(args: &mut RunArgs, matches: &ArgMatches) -> Result().ok(); let local_tls = apply_runtime_defaults(args)?; let guest_tls = local_tls.as_ref().map(GuestTlsPaths::from); @@ -259,7 +272,8 @@ fn prepare_server_config(args: &mut RunArgs, matches: &ArgMatches) -> Result Result Result Result<()> { - let prepared = prepare_server_config(&mut args, &matches)?; - let compute_driver = configured_compute_driver_for_startup(&prepared)?; +async fn run_from_args( + mut args: RunArgs, + matches: ArgMatches, + compute_drivers: ComputeDriverRegistry, +) -> Result<()> { + let prepared = prepare_server_config(&mut args, &matches, &compute_drivers)?; + let compute_driver = configured_compute_driver_for_startup(&compute_drivers, &prepared)?; let tracing_log_bus = TracingLogBus::new(); let otlp_config = prepared @@ -779,17 +793,9 @@ fn normalize_compute_driver_socket_args(args: &mut RunArgs, matches: &ArgMatches } } -fn effective_single_driver(args: &RunArgs) -> Option { - match args.drivers.as_slice() { - [] => openshell_core::config::detect_driver(), - [driver] => driver.parse().ok(), - _ => None, - } -} - -fn is_singleplayer_driver(args: &RunArgs) -> bool { +fn is_singleplayer_driver(driver: Option) -> bool { matches!( - effective_single_driver(args), + driver, Some(ComputeDriverKind::Docker | ComputeDriverKind::Podman | ComputeDriverKind::Vm) ) } @@ -798,6 +804,7 @@ fn resolve_mtls_auth_enabled( args: &RunArgs, matches: &ArgMatches, file: Option<&ConfigFile>, + compute_driver: Option, ) -> bool { let file_configured = file .and_then(|f| f.openshell.gateway.mtls_auth.as_ref()) @@ -810,7 +817,7 @@ fn resolve_mtls_auth_enabled( return false; } - is_singleplayer_driver(args) + is_singleplayer_driver(compute_driver) } #[cfg(test)] @@ -819,6 +826,43 @@ mod tests { use crate::TEST_ENV_LOCK as ENV_LOCK; use clap::Parser; use std::net::{IpAddr, Ipv4Addr}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + static REGISTRY_DETECTION_CALLS: AtomicUsize = AtomicUsize::new(0); + + fn detect_registered_docker() -> bool { + REGISTRY_DETECTION_CALLS.fetch_add(1, Ordering::SeqCst); + true + } + + #[derive(Clone, Copy)] + struct TestComputeDriverFactory; + + #[async_trait::async_trait] + impl crate::ComputeDriverFactory for TestComputeDriverFactory { + async fn build( + &self, + _context: crate::ComputeDriverBuildContext<'_>, + ) -> openshell_core::Result { + unreachable!("configuration tests do not construct the driver") + } + } + + fn detected_docker_registry() -> crate::ComputeDriverRegistry { + let mut registry = crate::ComputeDriverRegistry::new(); + registry + .install( + crate::ComputeDriverRegistration::new( + "docker", + 100, + Some(detect_registered_docker), + TestComputeDriverFactory, + ) + .unwrap(), + ) + .unwrap(); + registry + } struct EnvVarGuard { key: &'static str, @@ -1286,7 +1330,47 @@ mod tests { "/tmp/ca.crt", ]); - assert!(super::resolve_mtls_auth_enabled(&args, &matches, None)); + assert!(super::resolve_mtls_auth_enabled( + &args, + &matches, + None, + Some(openshell_core::ComputeDriverKind::Docker) + )); + } + + #[test] + fn registry_detection_drives_auth_defaults_once() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let state = tempfile::tempdir().unwrap(); + let config = tempfile::tempdir().unwrap(); + let _state = EnvVarGuard::set("XDG_STATE_HOME", state.path().to_str().unwrap()); + let _config = EnvVarGuard::set("XDG_CONFIG_HOME", config.path().to_str().unwrap()); + let _kubernetes = EnvVarGuard::set("KUBERNETES_SERVICE_HOST", "10.0.0.1"); + let _mtls = EnvVarGuard::remove("OPENSHELL_ENABLE_MTLS_AUTH"); + let _drivers = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + REGISTRY_DETECTION_CALLS.store(0, Ordering::SeqCst); + + let (mut args, matches) = parse_with_args(&[ + "openshell-gateway", + "--db-url", + "sqlite::memory:", + "--tls-cert", + "/tmp/server.crt", + "--tls-key", + "/tmp/server.key", + "--tls-client-ca", + "/tmp/ca.crt", + ]); + let registry = detected_docker_registry(); + + let prepared = super::prepare_server_config(&mut args, &matches, ®istry).unwrap(); + + assert_eq!(prepared.compute_driver.name(), "docker"); + assert!(prepared.config.compute_drivers.is_empty()); + assert!(prepared.config.mtls_auth.enabled); + assert_eq!(REGISTRY_DETECTION_CALLS.load(Ordering::SeqCst), 1); } #[test] @@ -1310,7 +1394,12 @@ mod tests { "/tmp/ca.crt", ]); - assert!(!super::resolve_mtls_auth_enabled(&args, &matches, None)); + assert!(!super::resolve_mtls_auth_enabled( + &args, + &matches, + None, + Some(openshell_core::ComputeDriverKind::Kubernetes) + )); } #[test] @@ -1345,7 +1434,8 @@ enabled = false assert!(!super::resolve_mtls_auth_enabled( &args, &matches, - Some(&file) + Some(&file), + Some(openshell_core::ComputeDriverKind::Docker) )); } @@ -1573,37 +1663,21 @@ ssh_session_ttl_secs = 1234 #[test] fn singleplayer_driver_matches_only_one_local_driver() { - for driver in ["docker", "podman", "vm"] { - let (args, _) = parse_with_args(&[ - "openshell-gateway", - "--db-url", - "sqlite::memory:", - "--drivers", - driver, - ]); + for driver in [ + openshell_core::ComputeDriverKind::Docker, + openshell_core::ComputeDriverKind::Podman, + openshell_core::ComputeDriverKind::Vm, + ] { assert!( - super::is_singleplayer_driver(&args), + super::is_singleplayer_driver(Some(driver)), "{driver} should be singleplayer" ); } - let (k8s, _) = parse_with_args(&[ - "openshell-gateway", - "--db-url", - "sqlite::memory:", - "--drivers", - "kubernetes", - ]); - assert!(!super::is_singleplayer_driver(&k8s)); - - let (multi, _) = parse_with_args(&[ - "openshell-gateway", - "--db-url", - "sqlite::memory:", - "--drivers", - "docker,podman", - ]); - assert!(!super::is_singleplayer_driver(&multi)); + assert!(!super::is_singleplayer_driver(Some( + openshell_core::ComputeDriverKind::Kubernetes + ))); + assert!(!super::is_singleplayer_driver(None)); } #[test] @@ -1629,7 +1703,11 @@ ssh_session_ttl_secs = 1234 Some(std::path::Path::new("/run/openshell/kyma.sock")) ); assert_eq!(args.drivers, ["kyma"]); - assert!(super::effective_single_driver(&args).is_none()); + assert!( + args.drivers[0] + .parse::() + .is_err() + ); } #[test] @@ -1809,8 +1887,12 @@ mem_mib = "not-a-number" "--disable-tls", ]); - let prepared = - super::prepare_server_config(&mut args, &matches).expect("server config is prepared"); + let prepared = super::prepare_server_config( + &mut args, + &matches, + &crate::install_default_compute_drivers(), + ) + .expect("server config is prepared"); assert_eq!(prepared.config.compute_drivers, vec!["podman".to_string()]); assert_eq!( diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index 9f4cac9a01..f0eb6f98b4 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -24,6 +24,12 @@ pub struct GuestTlsPaths { key: PathBuf, } +impl GuestTlsPaths { + pub(crate) fn as_paths(&self) -> (&std::path::Path, &std::path::Path, &std::path::Path) { + (&self.ca, &self.cert, &self.key) + } +} + impl From<&LocalTlsPaths> for GuestTlsPaths { fn from(paths: &LocalTlsPaths) -> Self { Self { @@ -60,7 +66,10 @@ pub struct RemoteDriverConfig { pub socket_path: PathBuf, } -fn driver_config_from_context(context: DriverStartupContext<'_>, driver_name: &str) -> Result +pub fn driver_config_from_context( + context: DriverStartupContext<'_>, + driver_name: &str, +) -> Result where T: Default + serde::de::DeserializeOwned, { diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index fe6bf62c91..2ba9f96cba 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -77,8 +77,9 @@ use tonic::{Code, Request, Status}; use tower::service_fn; use tracing::{Instrument as _, debug, info, warn}; -type DriverWatchStream = Pin> + Send>>; -type SharedComputeDriver = +pub type DriverWatchStream = + Pin> + Send>>; +pub type SharedComputeDriver = Arc + Send + Sync>; use traced_driver::TracedDriver; @@ -585,7 +586,7 @@ impl ComputeRuntime { driver.name = %driver_name, ) )] - async fn from_driver( + pub(crate) async fn from_driver( driver_name: String, driver: SharedComputeDriver, driver_process: Option>, diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 218e9396f0..0547714809 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -9,19 +9,9 @@ //! - Protocol multiplexing (gRPC + HTTP on same port) //! - mTLS support //! -//! TODO(driver-abstraction): `build_compute_runtime` still switches on -//! built-in driver names and calls driver-specific constructors -//! ([`ComputeRuntime::new_kubernetes`], [`ComputeRuntime::new_docker`], -//! [`compute::vm::spawn`] + [`ComputeRuntime::new_remote_driver`], -//! [`ComputeRuntime::new_podman`]). Endpoint-backed drivers now share the -//! remote `compute_driver.proto` path, so new remote drivers should enter -//! through named endpoint acquisition rather than gateway-wide socket side -//! channels. Once we have a generalized compute-driver registry, the remaining -//! per-arm wiring here should collapse to driver construction records that -//! produce either an in-process `SharedComputeDriver` or an acquired remote -//! endpoint, then hand the rest of the gateway a uniform [`ComputeRuntime`]. -//! The VM launch plumbing now lives in [`compute::vm`]; keep this file limited -//! to selecting and acquiring drivers. +//! Compiled-in compute drivers are installed into a registry at gateway +//! startup. Runtime selection only consults that registry or a configured +//! external endpoint; it does not switch on driver names. mod auth; pub mod certgen; @@ -58,8 +48,10 @@ mod tracing_setup; mod ws_tunnel; use metrics_exporter_prometheus::PrometheusBuilder; +#[cfg(target_os = "windows")] +use openshell_core::ComputeDriverKind; use openshell_core::net::set_tcp_nodelay_best_effort; -use openshell_core::{ComputeDriverKind, Config, Error, ObjectLabels, Result}; +use openshell_core::{Config, Error, ObjectLabels, Result}; use openshell_extension_core::{ BearerTokenSlot, ExtensionAudience, ExtensionCallerKind, ExtensionKind, MAX_EXTENSION_TOKEN_TTL, }; @@ -67,6 +59,7 @@ use openshell_supervisor_middleware::MiddlewareRegistry; use std::collections::{BTreeMap, HashMap}; use std::io::ErrorKind; use std::net::SocketAddr; +use std::path::Path; #[cfg(test)] use std::sync::LazyLock; use std::sync::{Arc, Mutex}; @@ -249,6 +242,7 @@ pub(crate) struct ServerStartupConfig { pub config: Config, pub config_file: Option, pub guest_tls: Option, + pub compute_driver: ComputeDriverSelection, } /// Server state shared across handlers. @@ -444,6 +438,7 @@ pub(crate) async fn run_server( config, config_file, guest_tls, + compute_driver: _, } = startup; let (shutdown_tx, shutdown_rx) = watch::channel(false); @@ -1081,10 +1076,465 @@ fn unsupported_builtin_compute_driver(driver: ComputeDriverKind) -> compute::Com )) } -// Internal wiring helper: each argument is a distinct piece of runtime state -// that must be passed through, so the count is justified. -#[allow(clippy::too_many_arguments)] type OperatorAllowlistArc = Option; +pub use compute::{DriverWatchStream, SharedComputeDriver}; + +/// Opaque result returned by a compiled compute-driver factory. +pub struct ComputeDriverBuildOutput { + runtime: ComputeRuntime, + operator_allowlist: OperatorAllowlistArc, +} + +/// Factory for a compute driver linked into a gateway binary. +#[async_trait::async_trait] +pub trait ComputeDriverFactory: Send + Sync { + async fn build( + &self, + context: ComputeDriverBuildContext<'_>, + ) -> Result; +} + +/// One named compiled-driver registration. +#[derive(Clone)] +pub struct ComputeDriverRegistration { + name: String, + detection_priority: u16, + detect: Option bool>, + factory: Arc, +} + +impl std::fmt::Debug for ComputeDriverRegistration { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ComputeDriverRegistration") + .field("name", &self.name) + .field("detection_priority", &self.detection_priority) + .field("has_detection_probe", &self.detect.is_some()) + .finish_non_exhaustive() + } +} + +impl ComputeDriverRegistration { + /// Define a compiled driver. Lower detection priorities are preferred. + pub fn new( + name: impl Into, + detection_priority: u16, + detect: Option bool>, + factory: impl ComputeDriverFactory + 'static, + ) -> Result { + let name = openshell_core::config::normalize_compute_driver_name(&name.into()) + .map_err(Error::config)?; + Ok(Self { + name, + detection_priority, + detect, + factory: Arc::new(factory), + }) + } +} + +/// Registry of compute drivers compiled into this gateway binary. +/// +/// Like `SQLx`'s `Any` driver registry, installation is explicit at the binary +/// composition boundary while runtime selection is generic. +#[derive(Clone, Default)] +pub struct ComputeDriverRegistry { + drivers: BTreeMap, +} + +#[derive(Clone, Debug)] +struct ComputeDriverDetection { + available: Vec, +} + +impl ComputeDriverDetection { + fn selected(&self) -> Option<&str> { + self.available.first().map(String::as_str) + } +} + +#[derive(Clone, Debug)] +pub(crate) enum ComputeDriverSelection { + Configured { name: String }, + AutoDetected(ComputeDriverDetection), +} + +impl ComputeDriverSelection { + pub(crate) fn name(&self) -> &str { + match self { + Self::Configured { name } => name, + Self::AutoDetected(detection) => detection + .selected() + .expect("auto-detected selection has an available driver"), + } + } +} + +impl ComputeDriverRegistry { + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Install a compiled driver factory. + pub fn install(&mut self, registration: ComputeDriverRegistration) -> Result<()> { + let name = registration.name.clone(); + match self.drivers.entry(name.clone()) { + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(registration); + Ok(()) + } + std::collections::btree_map::Entry::Occupied(_) => Err(Error::config(format!( + "compute driver '{name}' registered twice" + ))), + } + } + + /// Names installed into this gateway binary, in lexical order. + pub fn installed_driver_names(&self) -> impl Iterator { + self.drivers.keys().map(String::as_str) + } + + fn get(&self, name: &str) -> Option<&ComputeDriverRegistration> { + self.drivers.get(name) + } + + fn detect(&self) -> ComputeDriverDetection { + let mut candidates = self + .drivers + .values() + .filter(|registration| registration.detect.is_some()) + .collect::>(); + candidates.sort_by(|left, right| { + left.detection_priority + .cmp(&right.detection_priority) + .then_with(|| left.name.cmp(&right.name)) + }); + let available = candidates + .into_iter() + .filter(|registration| registration.detect.is_some_and(|detect| detect())) + .map(|registration| registration.name.clone()) + .collect(); + ComputeDriverDetection { available } + } + + pub(crate) fn select(&self, configured_drivers: &[String]) -> Result { + match configured_drivers { + [] => { + let detection = self.detect(); + if detection.selected().is_none() { + return Err(Error::config( + "no compute driver configured and auto-detection found no suitable installed \ + driver; set --drivers or OPENSHELL_DRIVERS=", + )); + } + Ok(ComputeDriverSelection::AutoDetected(detection)) + } + [driver] => { + let name = openshell_core::config::normalize_compute_driver_name(driver) + .map_err(Error::config)?; + Ok(ComputeDriverSelection::Configured { name }) + } + drivers => Err(Error::config(format!( + "multiple compute drivers are not supported yet; configured drivers: {}", + drivers.join(",") + ))), + } + } +} + +/// Install every first-party compute driver linked into the standard gateway. +#[must_use] +pub fn install_default_compute_drivers() -> ComputeDriverRegistry { + let mut registry = ComputeDriverRegistry::new(); + #[cfg(not(target_os = "windows"))] + { + registry + .install( + ComputeDriverRegistration::new( + "kubernetes", + 100, + Some(|| std::env::var_os("KUBERNETES_SERVICE_HOST").is_some()), + KubernetesComputeDriverFactory, + ) + .expect("valid kubernetes registration"), + ) + .expect("unique kubernetes registration"); + registry + .install( + ComputeDriverRegistration::new( + "podman", + 200, + Some(openshell_core::config::is_podman_available), + PodmanComputeDriverFactory, + ) + .expect("valid podman registration"), + ) + .expect("unique podman registration"); + registry + .install( + ComputeDriverRegistration::new( + "docker", + 300, + Some(openshell_core::config::is_docker_available), + DockerComputeDriverFactory, + ) + .expect("valid docker registration"), + ) + .expect("unique docker registration"); + registry + .install( + ComputeDriverRegistration::new("vm", u16::MAX, None, VmComputeDriverFactory) + .expect("valid vm registration"), + ) + .expect("unique vm registration"); + } + #[cfg(target_os = "windows")] + for name in ["kubernetes", "podman", "docker", "vm"] { + registry + .install( + ComputeDriverRegistration::new( + name, + u16::MAX, + None, + UnsupportedComputeDriverFactory, + ) + .expect("valid unsupported registration"), + ) + .expect("unique unsupported registration"); + } + registry +} + +pub struct ComputeDriverBuildContext<'a> { + driver_name: String, + config: &'a Config, + driver_startup: compute::driver_config::DriverStartupContext<'a>, + store: Arc, + sandbox_index: SandboxIndex, + sandbox_watch_bus: SandboxWatchBus, + tracing_log_bus: TracingLogBus, + supervisor_sessions: Arc, + shutdown_rx: watch::Receiver, +} + +impl ComputeDriverBuildContext<'_> { + #[must_use] + pub fn driver_name(&self) -> &str { + &self.driver_name + } + + #[must_use] + pub fn gateway_config(&self) -> &Config { + self.config + } + + #[must_use] + pub fn gateway_port(&self) -> u16 { + self.driver_startup.gateway_port + } + + #[must_use] + pub fn gateway_tls_enabled(&self) -> bool { + self.driver_startup.gateway_tls_enabled + } + + /// Gateway client credentials that a local driver may mount into guests. + #[must_use] + pub fn guest_tls_paths(&self) -> Option<(&Path, &Path, &Path)> { + self.driver_startup + .guest_tls + .map(compute::driver_config::GuestTlsPaths::as_paths) + } + + /// Deserialize the selected driver's merged TOML table. + pub fn driver_config(&self) -> Result + where + T: Default + serde::de::DeserializeOwned, + { + compute::driver_config::driver_config_from_context(self.driver_startup, &self.driver_name) + } + + #[must_use] + pub fn shutdown_receiver(&self) -> watch::Receiver { + self.shutdown_rx.clone() + } + + /// Finish construction of an in-process driver through the common runtime path. + pub async fn finish_in_process( + self, + driver: SharedComputeDriver, + ) -> Result { + let runtime = ComputeRuntime::from_driver( + self.driver_name, + driver, + None, + self.store, + self.sandbox_index, + self.sandbox_watch_bus, + self.tracing_log_bus, + self.supervisor_sessions, + ) + .await + .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; + Ok(ComputeDriverBuildOutput { + runtime, + operator_allowlist: None, + }) + } +} + +#[cfg(target_os = "windows")] +#[derive(Clone, Copy)] +struct UnsupportedComputeDriverFactory; + +#[cfg(target_os = "windows")] +#[async_trait::async_trait] +impl ComputeDriverFactory for UnsupportedComputeDriverFactory { + async fn build( + &self, + context: ComputeDriverBuildContext<'_>, + ) -> Result { + Err(Error::execution( + unsupported_builtin_compute_driver( + context + .driver_name + .parse() + .expect("default driver names are valid"), + ) + .to_string(), + )) + } +} + +#[cfg(not(target_os = "windows"))] +#[derive(Clone, Copy)] +struct KubernetesComputeDriverFactory; + +#[cfg(not(target_os = "windows"))] +#[async_trait::async_trait] +impl ComputeDriverFactory for KubernetesComputeDriverFactory { + async fn build( + &self, + context: ComputeDriverBuildContext<'_>, + ) -> Result { + warn_if_kubernetes_sandbox_jwt_expiry_disabled(context.config); + let config = compute::driver_config::builtin::kubernetes_config_from_context( + context.driver_startup, + )?; + let (runtime, operator_allowlist) = ComputeRuntime::new_kubernetes( + config, + context.store, + context.sandbox_index, + context.sandbox_watch_bus, + context.tracing_log_bus, + context.supervisor_sessions, + context.shutdown_rx, + ) + .await + .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; + Ok(ComputeDriverBuildOutput { + runtime, + operator_allowlist, + }) + } +} + +#[cfg(not(target_os = "windows"))] +#[derive(Clone, Copy)] +struct DockerComputeDriverFactory; + +#[cfg(not(target_os = "windows"))] +#[async_trait::async_trait] +impl ComputeDriverFactory for DockerComputeDriverFactory { + async fn build( + &self, + context: ComputeDriverBuildContext<'_>, + ) -> Result { + let driver_config = + compute::driver_config::builtin::docker_config_from_context(context.driver_startup)?; + let runtime = ComputeRuntime::new_docker( + context.config.clone(), + driver_config, + context.store, + context.sandbox_index, + context.sandbox_watch_bus, + context.tracing_log_bus, + context.supervisor_sessions, + ) + .await + .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; + Ok(ComputeDriverBuildOutput { + runtime, + operator_allowlist: None, + }) + } +} + +#[cfg(not(target_os = "windows"))] +#[derive(Clone, Copy)] +struct PodmanComputeDriverFactory; + +#[cfg(not(target_os = "windows"))] +#[async_trait::async_trait] +impl ComputeDriverFactory for PodmanComputeDriverFactory { + async fn build( + &self, + context: ComputeDriverBuildContext<'_>, + ) -> Result { + let driver_config = + compute::driver_config::builtin::podman_config_from_context(context.driver_startup)?; + let runtime = ComputeRuntime::new_podman( + driver_config, + context.store, + context.sandbox_index, + context.sandbox_watch_bus, + context.tracing_log_bus, + context.supervisor_sessions, + ) + .await + .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; + Ok(ComputeDriverBuildOutput { + runtime, + operator_allowlist: None, + }) + } +} + +#[cfg(not(target_os = "windows"))] +#[derive(Clone, Copy)] +struct VmComputeDriverFactory; + +#[cfg(not(target_os = "windows"))] +#[async_trait::async_trait] +impl ComputeDriverFactory for VmComputeDriverFactory { + async fn build( + &self, + context: ComputeDriverBuildContext<'_>, + ) -> Result { + let driver_config = + compute::driver_config::builtin::vm_config_from_context(context.driver_startup)?; + let otlp_config = context + .driver_startup + .file + .and_then(|file| file.openshell.gateway.otlp.as_ref()); + let endpoint = compute::vm::spawn(context.config, &driver_config, otlp_config).await?; + let runtime = ComputeRuntime::new_remote_driver( + endpoint, + context.store, + context.sandbox_index, + context.sandbox_watch_bus, + context.tracing_log_bus, + context.supervisor_sessions, + ) + .await + .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; + Ok(ComputeDriverBuildOutput { + runtime, + operator_allowlist: None, + }) + } +} #[allow(clippy::too_many_arguments)] async fn build_compute_runtime( @@ -1101,82 +1551,22 @@ async fn build_compute_runtime( info!(driver = %driver.name(), "Using compute driver"); let (runtime, operator_allowlist) = match driver { - #[cfg(target_os = "windows")] - ConfiguredComputeDriver::Builtin(driver) => { - return Err(Error::execution( - unsupported_builtin_compute_driver(driver).to_string(), - )); - } - #[cfg(not(target_os = "windows"))] - ConfiguredComputeDriver::Builtin(ComputeDriverKind::Kubernetes) => { - warn_if_kubernetes_sandbox_jwt_expiry_disabled(config); - let k8s_config = - compute::driver_config::builtin::kubernetes_config_from_context(driver_startup)?; - let (rt, allowlist) = ComputeRuntime::new_kubernetes( - k8s_config, - store, - sandbox_index, - sandbox_watch_bus, - tracing_log_bus, - supervisor_sessions.clone(), - shutdown_rx, - ) - .await - .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; - (rt, allowlist) - } - #[cfg(not(target_os = "windows"))] - ConfiguredComputeDriver::Builtin(ComputeDriverKind::Docker) => { - let docker_config = - compute::driver_config::builtin::docker_config_from_context(driver_startup)?; - let rt = ComputeRuntime::new_docker( - config.clone(), - docker_config, - store, - sandbox_index, - sandbox_watch_bus, - tracing_log_bus, - supervisor_sessions, - ) - .await - .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; - (rt, None) - } - #[cfg(not(target_os = "windows"))] - ConfiguredComputeDriver::Builtin(ComputeDriverKind::Podman) => { - let podman_config = - compute::driver_config::builtin::podman_config_from_context(driver_startup)?; - let rt = ComputeRuntime::new_podman( - podman_config, - store, - sandbox_index, - sandbox_watch_bus, - tracing_log_bus, - supervisor_sessions, - ) - .await - .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; - (rt, None) - } - #[cfg(not(target_os = "windows"))] - ConfiguredComputeDriver::Builtin(ComputeDriverKind::Vm) => { - let vm_config = - compute::driver_config::builtin::vm_config_from_context(driver_startup)?; - let otlp_config = driver_startup - .file - .and_then(|file| file.openshell.gateway.otlp.as_ref()); - let endpoint = compute::vm::spawn(config, &vm_config, otlp_config).await?; - let rt = ComputeRuntime::new_remote_driver( - endpoint, - store, - sandbox_index, - sandbox_watch_bus, - tracing_log_bus, - supervisor_sessions, - ) - .await - .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; - (rt, None) + ConfiguredComputeDriver::Registered(registration) => { + let output = registration + .factory + .build(ComputeDriverBuildContext { + driver_name: registration.name, + config, + driver_startup, + store, + sandbox_index, + sandbox_watch_bus, + tracing_log_bus, + supervisor_sessions, + shutdown_rx, + }) + .await?; + (output.runtime, output.operator_allowlist) } ConfiguredComputeDriver::Remote { name } => { let remote_config = @@ -1208,47 +1598,26 @@ async fn build_compute_runtime( #[derive(Debug, Clone)] pub(crate) enum ConfiguredComputeDriver { - Builtin(ComputeDriverKind), + Registered(ComputeDriverRegistration), Remote { name: String }, } impl ConfiguredComputeDriver { fn name(&self) -> &str { match self { - Self::Builtin(kind) => kind.as_str(), + Self::Registered(registration) => ®istration.name, Self::Remote { name } => name, } } } -fn configured_compute_driver( - config: &Config, - driver_startup: compute::driver_config::DriverStartupContext<'_>, -) -> Result { - match config.compute_drivers.as_slice() { - [] => match openshell_core::config::detect_driver() { - Some(ComputeDriverKind::Vm) => Err(Error::config( - "vm compute driver is opt-in only; set --drivers vm or OPENSHELL_DRIVERS=vm", - )), - Some(driver) => Ok(ConfiguredComputeDriver::Builtin(driver)), - None => Err(Error::config( - "no compute driver configured and auto-detection found no suitable driver; \ - set --drivers or OPENSHELL_DRIVERS to kubernetes, podman, docker, or vm", - )), - }, - [driver] => resolve_configured_compute_driver(driver, driver_startup), - drivers => Err(Error::config(format!( - "multiple compute drivers are not supported yet; configured drivers: {}", - drivers.join(",") - ))), - } -} - pub(crate) fn configured_compute_driver_for_startup( + registry: &ComputeDriverRegistry, startup: &ServerStartupConfig, ) -> Result { - configured_compute_driver( - &startup.config, + resolve_configured_compute_driver( + registry, + startup.compute_driver.name(), compute::driver_config::DriverStartupContext { file: startup.config_file.as_ref(), guest_tls: startup.guest_tls.as_ref(), @@ -1260,30 +1629,26 @@ pub(crate) fn configured_compute_driver_for_startup( } fn resolve_configured_compute_driver( + registry: &ComputeDriverRegistry, driver_name: &str, driver_startup: compute::driver_config::DriverStartupContext<'_>, ) -> Result { let name = openshell_core::config::normalize_compute_driver_name(driver_name) .map_err(Error::config)?; // An operator-provided endpoint replaces normal construction for the - // selected name. The gateway connects to it; it does not provision a - // remote implementation for canonical built-in names. + // selected name, including a compiled registration with the same name. + // The gateway connects to it; it does not provision the remote driver. if driver_startup.endpoint_overrides.contains_key(&name) { return Ok(ConfiguredComputeDriver::Remote { name }); } - let driver_kind = builtin_compute_driver(&name); - if let Some(kind) = driver_kind { - return Ok(ConfiguredComputeDriver::Builtin(kind)); + if let Some(registration) = registry.get(&name) { + return Ok(ConfiguredComputeDriver::Registered(registration.clone())); } Ok(ConfiguredComputeDriver::Remote { name }) } -fn builtin_compute_driver(name: &str) -> Option { - name.parse().ok() -} - fn kubernetes_sandbox_jwt_expiry_disabled(config: &Config) -> bool { config .gateway_jwt @@ -1364,9 +1729,8 @@ mod tests { BoundGatewayListener, ConfiguredComputeDriver, ConnectionProtocol, ExtensionKind, GatewayListenerScope, MultiplexService, ServerState, TlsAcceptor, allow_plaintext_service_http, bind_gateway_listeners, classify_initial_bytes, - configured_compute_driver, is_benign_tls_handshake_failure, - kubernetes_sandbox_jwt_expiry_disabled, mint_gateway_extension_credential, - serve_gateway_listener, + is_benign_tls_handshake_failure, kubernetes_sandbox_jwt_expiry_disabled, + mint_gateway_extension_credential, serve_gateway_listener, }; use openshell_core::{ ComputeDriverKind, Config, @@ -1375,7 +1739,7 @@ mod tests { use std::io::{Error, ErrorKind}; use std::net::SocketAddr; use std::sync::{ - Arc, + Arc, LazyLock, Mutex, atomic::{AtomicBool, Ordering}, }; use std::time::Duration; @@ -1491,6 +1855,52 @@ mod tests { } } + fn test_compute_drivers() -> super::ComputeDriverRegistry { + super::install_default_compute_drivers() + } + + fn select_compute_driver( + registry: &super::ComputeDriverRegistry, + config: &Config, + driver_startup: crate::compute::driver_config::DriverStartupContext<'_>, + ) -> openshell_core::Result { + let selection = registry.select(&config.compute_drivers)?; + super::resolve_configured_compute_driver(registry, selection.name(), driver_startup) + } + + #[derive(Clone, Copy)] + struct TestComputeDriverFactory; + + static DETECTION_PROBE_ORDER: LazyLock>> = + LazyLock::new(|| Mutex::new(Vec::new())); + + fn record_detection_probe(name: &'static str, available: bool) -> bool { + DETECTION_PROBE_ORDER.lock().unwrap().push(name); + available + } + + fn unavailable_first_probe() -> bool { + record_detection_probe("first", false) + } + + fn available_second_probe() -> bool { + record_detection_probe("second", true) + } + + fn available_third_probe() -> bool { + record_detection_probe("third", true) + } + + #[async_trait::async_trait] + impl super::ComputeDriverFactory for TestComputeDriverFactory { + async fn build( + &self, + _context: super::ComputeDriverBuildContext<'_>, + ) -> openshell_core::Result { + unreachable!("selection tests do not construct the driver") + } + } + fn test_tls_acceptor() -> (TempDir, TlsAcceptor) { install_rustls_provider(); @@ -1804,18 +2214,21 @@ mod tests { // Empty drivers triggers auto-detection, which may return Some or None // depending on the environment. This test verifies the auto-detection path // is taken rather than immediately returning an error. - let result = configured_compute_driver(&config, test_driver_startup(&config, None)); + let result = select_compute_driver( + &test_compute_drivers(), + &config, + test_driver_startup(&config, None), + ); // Either we get a detected driver or an error about none being detected. match result { - Ok(ConfiguredComputeDriver::Builtin(driver)) => { + Ok(ConfiguredComputeDriver::Registered(registration)) => { assert!( matches!( - driver, - ComputeDriverKind::Kubernetes - | ComputeDriverKind::Docker - | ComputeDriverKind::Podman + registration.name.as_str(), + "kubernetes" | "docker" | "podman" ), - "auto-detected unexpected driver: {driver:?}" + "auto-detected unexpected driver: {}", + registration.name ); } Ok(ConfiguredComputeDriver::Remote { name }) => { @@ -1824,19 +2237,81 @@ mod tests { Err(e) => { assert!( e.to_string() - .contains("auto-detection found no suitable driver"), + .contains("auto-detection found no suitable installed driver"), "unexpected error: {e}" ); } } } + #[test] + fn registry_detection_reports_available_drivers_in_priority_order() { + DETECTION_PROBE_ORDER.lock().unwrap().clear(); + let mut registry = super::ComputeDriverRegistry::new(); + registry + .install( + super::ComputeDriverRegistration::new( + "third", + 300, + Some(available_third_probe), + TestComputeDriverFactory, + ) + .unwrap(), + ) + .unwrap(); + registry + .install( + super::ComputeDriverRegistration::new( + "first", + 100, + Some(unavailable_first_probe), + TestComputeDriverFactory, + ) + .unwrap(), + ) + .unwrap(); + registry + .install( + super::ComputeDriverRegistration::new( + "second", + 200, + Some(available_second_probe), + TestComputeDriverFactory, + ) + .unwrap(), + ) + .unwrap(); + + let detection = registry.detect(); + assert_eq!(detection.selected(), Some("second")); + assert_eq!( + detection + .available + .iter() + .map(String::as_str) + .collect::>(), + vec!["second", "third"] + ); + assert_eq!( + DETECTION_PROBE_ORDER.lock().unwrap().as_slice(), + ["first", "second", "third"] + ); + assert_eq!( + registry.installed_driver_names().collect::>(), + vec!["first", "second", "third"] + ); + } + #[test] fn configured_compute_driver_rejects_multiple_entries() { let config = Config::new(None) .with_compute_drivers([ComputeDriverKind::Kubernetes, ComputeDriverKind::Podman]); - let err = - configured_compute_driver(&config, test_driver_startup(&config, None)).unwrap_err(); + let err = select_compute_driver( + &test_compute_drivers(), + &config, + test_driver_startup(&config, None), + ) + .unwrap_err(); assert!( err.to_string() .contains("multiple compute drivers are not supported yet") @@ -1847,33 +2322,45 @@ mod tests { #[test] fn configured_compute_driver_accepts_podman() { let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Podman]); - let driver = - configured_compute_driver(&config, test_driver_startup(&config, None)).unwrap(); + let driver = select_compute_driver( + &test_compute_drivers(), + &config, + test_driver_startup(&config, None), + ) + .unwrap(); assert!(matches!( driver, - ConfiguredComputeDriver::Builtin(ComputeDriverKind::Podman) + ConfiguredComputeDriver::Registered(registration) if registration.name == "podman" )); } #[test] fn configured_compute_driver_accepts_vm() { let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Vm]); - let driver = - configured_compute_driver(&config, test_driver_startup(&config, None)).unwrap(); + let driver = select_compute_driver( + &test_compute_drivers(), + &config, + test_driver_startup(&config, None), + ) + .unwrap(); assert!(matches!( driver, - ConfiguredComputeDriver::Builtin(ComputeDriverKind::Vm) + ConfiguredComputeDriver::Registered(registration) if registration.name == "vm" )); } #[test] fn configured_compute_driver_accepts_docker() { let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Docker]); - let driver = - configured_compute_driver(&config, test_driver_startup(&config, None)).unwrap(); + let driver = select_compute_driver( + &test_compute_drivers(), + &config, + test_driver_startup(&config, None), + ) + .unwrap(); assert!(matches!( driver, - ConfiguredComputeDriver::Builtin(ComputeDriverKind::Docker) + ConfiguredComputeDriver::Registered(registration) if registration.name == "docker" )); } @@ -1881,15 +2368,22 @@ mod tests { fn configured_compute_driver_resolves_named_remote() { let config = Config::new(None).with_compute_drivers(["kyma"]); - let driver = - configured_compute_driver(&config, test_driver_startup(&config, None)).unwrap(); + let driver = select_compute_driver( + &test_compute_drivers(), + &config, + test_driver_startup(&config, None), + ) + .unwrap(); match driver { ConfiguredComputeDriver::Remote { name } => { assert_eq!(name, "kyma"); } - ConfiguredComputeDriver::Builtin(other) => { - panic!("expected remote driver, got builtin driver {other:?}") + ConfiguredComputeDriver::Registered(other) => { + panic!( + "expected remote driver, got registered driver {}", + other.name + ) } } } @@ -1900,8 +2394,12 @@ mod tests { .with_compute_drivers([ComputeDriverKind::Vm]) .with_compute_driver_endpoint("vm", "/run/openshell/vm.sock"); - let driver = - configured_compute_driver(&config, test_driver_startup(&config, None)).unwrap(); + let driver = select_compute_driver( + &test_compute_drivers(), + &config, + test_driver_startup(&config, None), + ) + .unwrap(); assert!(matches!( driver, ConfiguredComputeDriver::Remote { name } if name == "vm" @@ -1914,8 +2412,12 @@ mod tests { .with_compute_drivers([ComputeDriverKind::Docker]) .with_compute_driver_endpoint("docker", "/run/openshell/docker.sock"); - let driver = - configured_compute_driver(&config, test_driver_startup(&config, None)).unwrap(); + let driver = select_compute_driver( + &test_compute_drivers(), + &config, + test_driver_startup(&config, None), + ) + .unwrap(); assert!(matches!( driver, ConfiguredComputeDriver::Remote { name } if name == "docker" diff --git a/crates/openshell-server/src/main.rs b/crates/openshell-server/src/main.rs index 0f33c685f4..c76761016d 100644 --- a/crates/openshell-server/src/main.rs +++ b/crates/openshell-server/src/main.rs @@ -7,5 +7,8 @@ use miette::Result; #[tokio::main] async fn main() -> Result<()> { - openshell_server::cli::run_cli().await + openshell_server::cli::run_cli_with_compute_drivers( + openshell_server::install_default_compute_drivers(), + ) + .await } diff --git a/crates/openshell-server/src/tracing_setup.rs b/crates/openshell-server/src/tracing_setup.rs index ac3c1ae79b..41b4e619ca 100644 --- a/crates/openshell-server/src/tracing_setup.rs +++ b/crates/openshell-server/src/tracing_setup.rs @@ -40,7 +40,7 @@ impl TracingHandle { pub fn podman_export_enabled(driver: &ConfiguredComputeDriver) -> bool { matches!( driver, - ConfiguredComputeDriver::Builtin(openshell_core::ComputeDriverKind::Podman) + ConfiguredComputeDriver::Registered(registration) if registration.name == "podman" ) } @@ -83,20 +83,22 @@ pub fn install( mod tests { use super::*; + #[cfg(not(target_os = "windows"))] #[test] fn podman_export_is_enabled_only_when_podman_is_selected() { - use crate::ConfiguredComputeDriver; - use openshell_core::ComputeDriverKind; + let registry = crate::install_default_compute_drivers(); + let registered = |name| { + ConfiguredComputeDriver::Registered( + registry + .get(name) + .unwrap_or_else(|| panic!("{name} driver is registered")) + .clone(), + ) + }; - assert!(podman_export_enabled(&ConfiguredComputeDriver::Builtin( - ComputeDriverKind::Podman - ))); - assert!(!podman_export_enabled(&ConfiguredComputeDriver::Builtin( - ComputeDriverKind::Docker - ))); - assert!(!podman_export_enabled(&ConfiguredComputeDriver::Builtin( - ComputeDriverKind::Kubernetes - ))); + assert!(podman_export_enabled(®istered("podman"))); + assert!(!podman_export_enabled(®istered("docker"))); + assert!(!podman_export_enabled(®istered("kubernetes"))); assert!(!podman_export_enabled(&ConfiguredComputeDriver::Remote { name: "custom".to_string(), }));