From 33517b32bd3fdacce0ce8a0d862253aa10f0504f Mon Sep 17 00:00:00 2001 From: Bryce Wilkinson <22760097+bjw123@users.noreply.github.com> Date: Thu, 20 Aug 2026 09:17:32 +0000 Subject: [PATCH] feat(server): make the database pool ceiling configurable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both stores hardcoded their pool ceiling: 10 connections for Postgres, 5 for on-disk SQLite. One pool is shared by every database-backed RPC, so that number is also the gateway's ceiling on concurrent database work. Once every connection is checked out, callers queue on acquire and sqlx logs "time to acquire exceeded slow threshold"; sandbox creates then time out and are retried, which adds load rather than shedding it. The right ceiling is deployment-specific — it depends on how many gateway replicas share the database and what max_connections the server itself allows — so it cannot be a single number baked into the binary. Expose it on the same config surface as the rest of the gateway's settings: --db-max-connections, OPENSHELL_DB_MAX_CONNECTIONS, and the TOML key database_max_connections, resolved in that precedence order and rendered by the Helm chart from server.dbMaxConnections. Omitting it keeps each backend's previous value, so existing deployments do not move. Values below 1 are rejected rather than silently replaced by the default: a zero pool would block every acquire, and quietly ignoring a typo would reproduce the ceiling the operator is trying to lift. An in-memory SQLite database stays pinned to one connection, since the database lives inside that connection. Refs: #2561 Signed-off-by: Bryce Wilkinson <22760097+bjw123@users.noreply.github.com> --- crates/openshell-core/src/config.rs | 20 +++++ crates/openshell-server/src/cli.rs | 90 +++++++++++++++++++ crates/openshell-server/src/config_file.rs | 47 ++++++++++ crates/openshell-server/src/lib.rs | 4 +- .../openshell-server/src/persistence/mod.rs | 20 ++++- .../src/persistence/postgres.rs | 15 +++- .../src/persistence/sqlite.rs | 40 ++++++++- .../openshell-server/src/persistence/tests.rs | 31 +++++++ deploy/helm/openshell/README.md | 1 + .../openshell/templates/gateway-config.yaml | 7 ++ .../openshell/tests/gateway_config_test.yaml | 24 +++++ deploy/helm/openshell/values.yaml | 6 ++ deploy/man/openshell-gateway.8.md | 8 ++ docs/kubernetes/setup.mdx | 1 + docs/reference/gateway-config.mdx | 22 +++++ 15 files changed, 327 insertions(+), 9 deletions(-) diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index fcbdeb73b6..1afe6c4911 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -461,6 +461,16 @@ pub struct Config { /// Database URL for persistence. pub database_url: String, + /// Connection ceiling for the persistence pool. + /// + /// `None` leaves the backend's built-in default in place. The right + /// ceiling depends on the deployment — how many gateway replicas share + /// the database, and what `max_connections` a Postgres server itself + /// allows — so it cannot be one number baked into the binary. An + /// in-memory `SQLite` database ignores it: the database lives in its single + /// connection. + pub database_max_connections: Option, + /// Compute drivers configured for the gateway. /// /// The config shape allows multiple drivers so the gateway can evolve @@ -835,6 +845,7 @@ impl Config { mtls_auth: MtlsAuthConfig::default(), gateway_jwt: None, database_url: String::new(), + database_max_connections: None, compute_drivers: vec![], compute_driver_endpoints: BTreeMap::new(), credential_drivers: Vec::new(), @@ -879,6 +890,15 @@ impl Config { self } + /// Create a new configuration with a database pool connection ceiling. + /// + /// `None` keeps the persistence backend's default. + #[must_use] + pub const fn with_database_max_connections(mut self, max_connections: Option) -> Self { + self.database_max_connections = max_connections; + self + } + /// Create a new configuration with the configured compute drivers. #[must_use] pub fn with_compute_drivers(mut self, drivers: I) -> Self diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 2e86c3a1b5..481f184797 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -94,6 +94,20 @@ struct RunArgs { #[arg(long, env = "OPENSHELL_DB_URL")] db_url: Option, + /// Connection ceiling for the database pool, shared by every + /// database-backed request. + /// + /// When unset, the gateway uses its backend default: 10 with Postgres, + /// 5 with an on-disk `SQLite` database. An in-memory `SQLite` database is + /// always a single connection. Each replica opens its own pool, so keep + /// the total below what the database server itself admits. + #[arg( + long, + env = "OPENSHELL_DB_MAX_CONNECTIONS", + value_parser = clap::value_parser!(u32).range(1..) + )] + db_max_connections: Option, + /// Compute drivers configured for this gateway. /// /// Accepts a comma-delimited list such as `kubernetes` or @@ -383,6 +397,7 @@ fn prepare_server_config(args: &mut RunArgs, matches: &ArgMatches) -> Result, window_seconds: Option) -> Result<()> { @@ -1489,6 +1510,75 @@ grpc_rate_limit_window_seconds = 30 assert_eq!(args.grpc_rate_limit_window_seconds, Some(30)); } + #[test] + fn file_db_max_connections_populates_args_when_cli_omits() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _g = EnvVarGuard::remove("OPENSHELL_DB_MAX_CONNECTIONS"); + + let (mut args, matches) = + parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + assert_eq!( + args.db_max_connections, None, + "default leaves the pool alone" + ); + let file = config_file_from_toml( + r" +[openshell.gateway] +database_max_connections = 64 +", + ); + merge_file_into_args(&mut args, &file.openshell.gateway, &matches); + + assert_eq!(args.db_max_connections, Some(64)); + } + + #[test] + fn env_db_max_connections_overrides_file_value() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _g = EnvVarGuard::set("OPENSHELL_DB_MAX_CONNECTIONS", "128"); + + let (mut args, matches) = + parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + let file = config_file_from_toml( + r" +[openshell.gateway] +database_max_connections = 64 +", + ); + merge_file_into_args(&mut args, &file.openshell.gateway, &matches); + + assert_eq!(args.db_max_connections, Some(128)); + } + + #[test] + fn db_max_connections_rejects_a_zero_or_unparseable_ceiling() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _g = EnvVarGuard::remove("OPENSHELL_DB_MAX_CONNECTIONS"); + + // Zero would deadlock every `acquire`, and a silent fallback to the + // default would reproduce the ceiling the operator is trying to lift. + for bad in ["0", "-5", "many"] { + assert!( + command() + .try_get_matches_from([ + "openshell-gateway", + "--db-url", + "sqlite::memory:", + "--db-max-connections", + bad, + ]) + .is_err(), + "input {bad:?} must be rejected" + ); + } + } + #[test] fn aux_listener_preserves_file_ip_against_public_bind() { use std::net::SocketAddr; diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 00b7a2f64d..0e86a9b250 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -99,6 +99,17 @@ pub struct GatewayFileSection { #[serde(default)] pub log_level: Option, + // ── Database ───────────────────────────────────────────────────────── + /// Connection ceiling for the persistence pool. One shared pool serves + /// every database-backed RPC, so this is also the gateway's ceiling on + /// concurrent database work. On Postgres, total server connections are + /// `database_max_connections × replica_count`, which must stay under the + /// server's own `max_connections`. Omit to keep the backend default (10 + /// for Postgres, 5 for on-disk `SQLite`); an in-memory `SQLite` database + /// is always a single connection. + #[serde(default)] + pub database_max_connections: Option, + // ── Drivers ────────────────────────────────────────────────────────── #[serde(default)] pub compute_drivers: Option>, @@ -399,6 +410,14 @@ pub fn load(path: &Path) -> Result { cli: "--db-url", }); } + // A zero-sized pool would deadlock every `acquire` rather than degrade, + // so reject it at load instead of starting a gateway that cannot serve. + if file.openshell.gateway.database_max_connections == Some(0) { + return Err(ConfigFileError::InvalidValue { + field: "openshell.gateway.database_max_connections", + message: "must be greater than zero; omit the key to use the built-in default", + }); + } if file .openshell .gateway @@ -912,6 +931,34 @@ database_url = "sqlite::memory:" )); } + #[test] + fn parses_database_max_connections() { + let toml = r" +[openshell.gateway] +database_max_connections = 64 +"; + let tmp = write_tmp(toml); + let file = load(tmp.path()).expect("a positive pool ceiling parses"); + assert_eq!(file.openshell.gateway.database_max_connections, Some(64)); + } + + #[test] + fn rejects_zero_database_max_connections() { + let toml = r" +[openshell.gateway] +database_max_connections = 0 +"; + let tmp = write_tmp(toml); + let err = load(tmp.path()).expect_err("a zero pool ceiling must be rejected"); + assert!(matches!( + err, + ConfigFileError::InvalidValue { + field: "openshell.gateway.database_max_connections", + .. + } + )); + } + #[test] fn rejects_unknown_gateway_field() { let toml = r" diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index a6031a9bc6..137bb8bca1 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -554,7 +554,9 @@ pub(crate) async fn run_server( .map_err(|error| Error::config(format!("middleware registration failed: {error}")))?, ); - let store = Arc::new(Store::connect(database_url).await?); + let store = Arc::new( + Store::connect_with_pool_size(database_url, config.database_max_connections).await?, + ); let credentials = credentials::CredentialRuntime::from_config_file_with_store( &config, config_file.as_ref(), diff --git a/crates/openshell-server/src/persistence/mod.rs b/crates/openshell-server/src/persistence/mod.rs index 3ad20e1082..56ce35fce4 100644 --- a/crates/openshell-server/src/persistence/mod.rs +++ b/crates/openshell-server/src/persistence/mod.rs @@ -201,10 +201,24 @@ impl Store { matches!(self, Self::Sqlite(_)) } - /// Connect to a persistence store based on the database URL. + /// Connect to a persistence store based on the database URL, leaving pool + /// sizing at the backend default. pub async fn connect(url: &str) -> CoreResult { + Self::connect_with_pool_size(url, None).await + } + + /// Connect to a persistence store based on the database URL. + /// + /// `max_connections` overrides the backend's pool ceiling, which stays at + /// its built-in default when `None`. The two backends have different + /// defaults, and an in-memory `SQLite` database ignores the override + /// entirely — it must be held by exactly one connection. + pub async fn connect_with_pool_size( + url: &str, + max_connections: Option, + ) -> CoreResult { if url.starts_with("postgres://") || url.starts_with("postgresql://") { - let store = PostgresStore::connect(url) + let store = PostgresStore::connect(url, max_connections) .await .map_err(|e| CoreError::execution(e.to_string()))?; store @@ -213,7 +227,7 @@ impl Store { .map_err(|e| CoreError::execution(e.to_string()))?; Ok(Self::Postgres(store)) } else if url.starts_with("sqlite:") { - let store = SqliteStore::connect(url) + let store = SqliteStore::connect(url, max_connections) .await .map_err(|e| CoreError::execution(e.to_string()))?; store diff --git a/crates/openshell-server/src/persistence/postgres.rs b/crates/openshell-server/src/persistence/postgres.rs index 9195f5dda4..1781f2d000 100644 --- a/crates/openshell-server/src/persistence/postgres.rs +++ b/crates/openshell-server/src/persistence/postgres.rs @@ -25,10 +25,21 @@ pub struct PostgresStore { pool: PgPool, } +/// Pool ceiling used when the operator has not configured one. +/// +/// A single shared pool serves every database-backed RPC, so this value is +/// also the gateway's ceiling on concurrent database work. It is high enough +/// for a single-replica gateway against a default-sized server; fleets that +/// outgrow it raise it through `[openshell.gateway] database_max_connections`. +pub const DEFAULT_MAX_CONNECTIONS: u32 = 10; + impl PostgresStore { - pub async fn connect(url: &str) -> PersistenceResult { + /// Connect and size the pool, falling back to [`DEFAULT_MAX_CONNECTIONS`]. + pub async fn connect(url: &str, max_connections: Option) -> PersistenceResult { + let max_connections = max_connections.unwrap_or(DEFAULT_MAX_CONNECTIONS); + tracing::info!(max_connections, "sizing Postgres connection pool"); let pool = PgPoolOptions::new() - .max_connections(10) + .max_connections(max_connections) .connect(url) .await .map_err(|e| map_db_error(&e))?; diff --git a/crates/openshell-server/src/persistence/sqlite.rs b/crates/openshell-server/src/persistence/sqlite.rs index b54c41e111..caa0e42889 100644 --- a/crates/openshell-server/src/persistence/sqlite.rs +++ b/crates/openshell-server/src/persistence/sqlite.rs @@ -23,6 +23,14 @@ static SQLITE_MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("./migrations/s use super::{DRAFT_CHUNK_OBJECT_TYPE, POLICY_OBJECT_TYPE}; +/// Pool ceiling for an on-disk database when the operator has not configured +/// one. +/// +/// `SQLite` serializes writers, so raising this mainly buys read concurrency; +/// a gateway that needs more write throughput wants Postgres, not a larger +/// `SQLite` pool. +pub const DEFAULT_MAX_CONNECTIONS: u32 = 5; + #[derive(Debug, Clone)] pub struct SqliteStore { pool: SqlitePool, @@ -35,9 +43,35 @@ impl SqliteStore { self.pool.close().await; } - pub async fn connect(url: &str) -> PersistenceResult { + /// Test support only: the pool's effective connection ceiling. + #[cfg(test)] + pub(crate) fn max_connections_for_test(&self) -> u32 { + self.pool.options().get_max_connections() + } + + /// Connect and size the pool, falling back to [`DEFAULT_MAX_CONNECTIONS`] + /// for an on-disk database. + /// + /// An in-memory database is always held by exactly one connection — the + /// database lives in that connection, so a second one would see a + /// different (empty) database. A configured ceiling is ignored there. + pub async fn connect(url: &str, max_connections: Option) -> PersistenceResult { let is_in_memory = url.contains(":memory:") || url.contains("mode=memory"); - let max_connections = if is_in_memory { 1 } else { 5 }; + let max_connections = if is_in_memory { + if max_connections.is_some_and(|configured| configured != 1) { + tracing::warn!( + "ignoring the configured database pool ceiling: an in-memory SQLite database is held by a single connection" + ); + } + 1 + } else { + max_connections.unwrap_or(DEFAULT_MAX_CONNECTIONS) + }; + // Keep the pool eagerly warm up to the built-in size, but let a raised + // ceiling grow on demand rather than pinning that many file handles + // open for the life of the process. + let min_connections = max_connections.min(DEFAULT_MAX_CONNECTIONS); + tracing::info!(max_connections, "sizing SQLite connection pool"); let options = SqliteConnectOptions::from_str(url) .map_err(|e| map_db_error(&e))? @@ -45,7 +79,7 @@ impl SqliteStore { let mut pool_options = SqlitePoolOptions::new() .max_connections(max_connections) - .min_connections(max_connections); + .min_connections(min_connections); if is_in_memory { pool_options = pool_options.idle_timeout(None).max_lifetime(None); diff --git a/crates/openshell-server/src/persistence/tests.rs b/crates/openshell-server/src/persistence/tests.rs index 6227eec297..edce4e628b 100644 --- a/crates/openshell-server/src/persistence/tests.rs +++ b/crates/openshell-server/src/persistence/tests.rs @@ -132,6 +132,37 @@ async fn sqlite_connect_runs_embedded_migrations() { assert!(records.is_empty()); } +#[tokio::test] +async fn sqlite_pool_ceiling_defaults_and_honours_an_override() { + use super::sqlite::{DEFAULT_MAX_CONNECTIONS, SqliteStore}; + + let tmp = tempfile::tempdir().expect("tempdir"); + let db_path = tmp.path().join("openshell.db"); + let url = format!("sqlite:{}?mode=rwc", db_path.display()); + + let unset = SqliteStore::connect(&url, None) + .await + .expect("connect to sqlite"); + assert_eq!(unset.max_connections_for_test(), DEFAULT_MAX_CONNECTIONS); + + let raised = SqliteStore::connect(&url, Some(32)) + .await + .expect("connect to sqlite"); + assert_eq!(raised.max_connections_for_test(), 32); +} + +/// An in-memory database lives inside its single connection: a second one +/// would open a different, empty database, so the ceiling cannot be raised. +#[tokio::test] +async fn in_memory_sqlite_pins_the_pool_to_one_connection() { + use super::sqlite::SqliteStore; + + let store = SqliteStore::connect("sqlite::memory:", Some(32)) + .await + .expect("connect to in-memory sqlite"); + assert_eq!(store.max_connections_for_test(), 1); +} + #[cfg(unix)] #[tokio::test] async fn sqlite_connect_restricts_db_file_permissions() { diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 93dab354b6..1a55a6cf14 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -226,6 +226,7 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | server.credentialDrivers.vault.timeoutSecs | string | `""` | HTTP request timeout in seconds. Empty = driver default. | | server.credentialDrivers.vault.tokenPath | string | `""` | Mounted token file path when authMethod is token_file. | | server.credentialStorage.existingSecret | string | `""` | Name of a pre-existing Secret containing the key-encryption key. When set, the chart does NOT generate a new Secret; it references this one instead. The Secret must contain a key named "key-encryption-key" with a base64-encoded 32-byte value. Required for GitOps workflows that render manifests with `helm template` (where `lookup` is unavailable). | +| server.dbMaxConnections | int | `0` | Connection ceiling for the gateway's database pool, shared by every database-backed request. 0 (default) uses the gateway's backend default: 10 for PostgreSQL, 5 for on-disk SQLite. On PostgreSQL each replica opens its own pool, so keep dbMaxConnections * replicaCount below the server's own max_connections. | | server.dbUrl | string | `"sqlite:/var/openshell/openshell.db"` | Gateway database URL (used for the default SQLite backend). | | server.defaultRuntimeClassName | string | `""` | Default Kubernetes runtimeClassName for sandbox pods. Applied when a CreateSandbox request does not specify one. Empty (default) = omit the field, using the cluster's default RuntimeClass. Set to a RuntimeClass name (e.g. "kata-containers", "nvidia") to apply it to all sandboxes that don't explicitly override it. | | server.disableTls | bool | `false` | Disable TLS entirely - the server listens on plaintext HTTP. Set to true when a reverse proxy / tunnel terminates TLS at the edge. | diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 9d24dbd917..a148e7e04e 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -42,6 +42,13 @@ data: metrics_bind_address = "0.0.0.0:{{ .Values.service.metricsPort }}" {{- end }} log_level = {{ .Values.server.logLevel | quote }} + {{- $dbMaxConnections := int .Values.server.dbMaxConnections }} + {{- if lt $dbMaxConnections 0 }} + {{- fail "server.dbMaxConnections must not be negative; set 0 to use the gateway default" }} + {{- end }} + {{- if gt $dbMaxConnections 0 }} + database_max_connections = {{ $dbMaxConnections }} + {{- end }} {{- if $credentialDrivers }} credential_drivers = [{{- range $i, $driver := $credentialDrivers }}{{ if $i }}, {{ end }}{{ $driver | quote }}{{- end }}] {{- end }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index afacd01eb4..2c2b1d71ea 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -349,6 +349,30 @@ tests: - failedTemplate: errorMessage: "server.grpcRateLimit.requests and server.grpcRateLimit.windowSeconds must not be negative; they map to unsigned gateway settings" + - it: omits the database pool ceiling by default so the gateway keeps its own default + template: templates/gateway-config.yaml + asserts: + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'database_max_connections\s*=' + + - it: renders the database pool ceiling under [openshell.gateway] when set + template: templates/gateway-config.yaml + set: + server.dbMaxConnections: 64 + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\].*?database_max_connections\s*=\s*64' + + - it: fails to render when dbMaxConnections is negative + template: templates/statefulset.yaml + set: + server.dbMaxConnections: -1 + asserts: + - failedTemplate: + errorMessage: "server.dbMaxConnections must not be negative; set 0 to use the gateway default" + - it: uses the configured existing sandbox service account name template: templates/gateway-config.yaml set: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 33337c768e..a752f73176 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -194,6 +194,12 @@ server: # from this Secret instead of using dbUrl. The Secret must contain a # `uri` key, e.g. postgresql://user:pass@host:5432/dbname. externalDbSecret: "" + # -- Connection ceiling for the gateway's database pool, shared by every + # database-backed request. 0 (default) uses the gateway's backend default: + # 10 for PostgreSQL, 5 for on-disk SQLite. On PostgreSQL each replica opens + # its own pool, so keep dbMaxConnections * replicaCount below the server's + # own max_connections. + dbMaxConnections: 0 # -- Default sandbox image used when requests do not specify one. sandboxImage: "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" # -- Kubernetes imagePullPolicy for sandbox pods. Empty = Kubernetes default diff --git a/deploy/man/openshell-gateway.8.md b/deploy/man/openshell-gateway.8.md index 2d584c4ba1..c2e51f025f 100644 --- a/deploy/man/openshell-gateway.8.md +++ b/deploy/man/openshell-gateway.8.md @@ -58,6 +58,14 @@ TLS. stores SQLite state under *~/.local/state/openshell/gateway/*. Environment: **OPENSHELL_DB_URL**. +**--db-max-connections** *N* +: Connection ceiling for the database pool, shared by every + database-backed request. Must be at least 1. When unset, the gateway + uses the backend default: **10** for Postgres, **5** for on-disk + SQLite. An in-memory SQLite database is always one connection. + TOML key: `database_max_connections`. + Environment: **OPENSHELL_DB_MAX_CONNECTIONS**. + **--drivers** *DRIVER*\[,*DRIVER*\] : Compute driver. Accepts a comma-delimited list. The gateway currently requires exactly one driver. Options: **podman**, diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index d305cb0f8a..0b4e0afd0d 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -151,6 +151,7 @@ The most commonly changed values are: | `workload.allowMultiReplicaStatefulSet` | Allow `replicaCount > 1` with `workload.kind=statefulset`. Prefer Deployment for external database-backed multi-replica gateways. | | `server.sandboxNamespace` | Namespace where sandbox pods are created. Defaults to the Helm release namespace when left empty. | | `server.externalDbSecret` | Secret containing a PostgreSQL connection URI in the `uri` key. Use when the database is managed outside the chart. | +| `server.dbMaxConnections` | Connection ceiling for the gateway's database pool. `0` keeps the gateway default (10 for PostgreSQL, 5 for on-disk SQLite). Each replica opens its own pool, so keep `dbMaxConnections × replicaCount` below the database server's `max_connections`. Refer to [Gateway Config](/reference/gateway-config). | | `server.telemetryEnabled` | Enable anonymous OpenShell telemetry from the gateway and its sandbox supervisors. Set to `false` to opt out. | | `server.sandboxImage` | Default sandbox image used when a sandbox does not specify one. | | `server.sandboxImagePullSecrets` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 8d74b3d44f..5503c565db 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -75,6 +75,10 @@ metrics_bind_address = "0.0.0.0:9090" log_level = "info" +# Connection ceiling for the gateway's database pool. Omit to use the backend +# default: 10 for Postgres, 5 for on-disk SQLite. +database_max_connections = 10 + # When empty, the gateway auto-detects Kubernetes, then Podman, then Docker. # VM is never auto-detected and requires an explicit entry here. compute_drivers = ["kubernetes"] @@ -210,6 +214,24 @@ Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth `[openshell.gateway.auth] allow_unauthenticated_users = true` is an unsafe local-development and trusted-proxy escape hatch. It accepts user-facing CLI/API calls without OIDC or mTLS credentials while sandbox supervisors still authenticate with gateway-minted sandbox JWTs. Leave it false for shared and production gateways. +## Database Pool + +`[openshell.gateway] database_max_connections` sets the connection ceiling for the gateway's database pool. One pool is shared by every database-backed RPC, so this value is also the gateway's ceiling on concurrent database work. When the pool is saturated, callers queue on connection acquisition and sqlx logs that the time to acquire a connection exceeded its slow threshold. The background readiness monitor draws from the same pool, so sustained saturation can push its database ping past the check timeout and flip `/readyz` to not-ready while the gateway is otherwise serving. + +Omit the key to use the backend default, which is unchanged from earlier releases: + +| Backend | Default ceiling | +|---|---| +| Postgres | 10 | +| SQLite (on disk) | 5 | +| SQLite (in memory) | 1, always. The database lives inside its single connection, so a configured value is ignored and the gateway logs that it was. | + +The equivalent flag and environment variable are `--db-max-connections` and `OPENSHELL_DB_MAX_CONNECTIONS`, and the Helm chart renders this key from `server.dbMaxConnections`. Values below `1` are rejected at startup: a zero-sized pool would block every acquisition rather than degrade. + +On Postgres, each replica opens its own pool, so plan for `database_max_connections × replicaCount` connections and keep that total below the server's own `max_connections` (and below any connection-pooler limit in front of it). Raising the gateway ceiling past what the server admits moves the failure from a slow acquire to a refused connection. + +On SQLite, writers are serialized regardless of pool size, so a larger ceiling buys read concurrency only. A gateway that needs more write throughput wants an external Postgres database (`server.externalDbSecret`), not a larger SQLite pool. + ## OTLP Export `[openshell.gateway.otlp]` enables OpenTelemetry export over OTLP/gRPC. Omit the table to disable export; there is no separate `enabled` flag.