From 1a8ff1562f455f9c65d638c376d66a82c568ccef Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 17:09:05 -0700 Subject: [PATCH 01/19] feat(podman): adopt isolated sandbox and supervisor containers Signed-off-by: Drew Newberry --- Cargo.lock | 3 + architecture/compute-runtimes.md | 9 + crates/openshell-driver-podman/Cargo.toml | 3 + crates/openshell-driver-podman/NETWORKING.md | 494 ++------------- crates/openshell-driver-podman/README.md | 583 +++-------------- crates/openshell-driver-podman/src/client.rs | 85 +++ .../openshell-driver-podman/src/container.rs | 589 +++++++++++------- crates/openshell-driver-podman/src/driver.rs | 578 +++++++++++++---- crates/openshell-driver-podman/src/grpc.rs | 9 +- .../openshell-driver-podman/src/isolation.rs | 381 +++++++++++ crates/openshell-driver-podman/src/lib.rs | 1 + .../openshell-driver-podman/src/test_utils.rs | 6 +- crates/openshell-driver-podman/src/watcher.rs | 138 +++- docs/reference/gateway-config.mdx | 263 ++------ e2e/rust/tests/podman_gateway_start.rs | 18 +- e2e/rust/tests/podman_oci_identity.rs | 64 +- e2e/with-podman-gateway.sh | 17 +- skills/debug-openshell-cluster/SKILL.md | 12 +- 18 files changed, 1743 insertions(+), 1510 deletions(-) create mode 100644 crates/openshell-driver-podman/src/isolation.rs diff --git a/Cargo.lock b/Cargo.lock index a4c7430b9c..27b831823c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4220,6 +4220,7 @@ dependencies = [ "miette", "nix 0.29.0", "openshell-core", + "openshell-isolation-interface", "openshell-otel", "openshell-otel-test-support", "opentelemetry", @@ -4228,6 +4229,7 @@ dependencies = [ "rustix 1.1.4", "serde", "serde_json", + "tar", "temp-env", "thiserror 2.0.18", "tokio", @@ -4238,6 +4240,7 @@ dependencies = [ "tracing-opentelemetry", "tracing-subscriber", "url", + "uuid", ] [[package]] diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 0889dc3e60..a2bb25c312 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -5,6 +5,15 @@ gateway. A supported runtime provisions `openshell-sandbox` inside the workload, `openshell-supervisor` outside it, a protected channel between them, and an independent outer network fence. Drivers do not implement policy evaluation. +Podman provisions a paired workload and supervisor container using its native +libpod API. The workload uses `network=none`; the external supervisor alone joins +the configured network. A per-sandbox named volume carries their mutually +authenticated gRPC Unix socket, with supervisor credentials kept in its separate +filesystem. Both containers run as the resolved non-root identity with all +capabilities dropped. They share only a user namespace for volume ownership, +not PID, mount, or network namespaces. Podman owns paired lifecycle and health; +the common protocol owns process, identity, TCP, DNS, and forwarding semantics. + ## Driver Contract Each runtime receives a sandbox spec and canonical policy from the gateway and diff --git a/crates/openshell-driver-podman/Cargo.toml b/crates/openshell-driver-podman/Cargo.toml index 8b3e014e8c..cd463b86fe 100644 --- a/crates/openshell-driver-podman/Cargo.toml +++ b/crates/openshell-driver-podman/Cargo.toml @@ -17,6 +17,9 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } openshell-otel = { path = "../openshell-otel" } +openshell-isolation-interface = { path = "../openshell-isolation-interface" } +tar = "0.4" +uuid = { workspace = true } tokio = { workspace = true } tonic = { workspace = true, features = ["transport"] } diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index 567abcbfcd..9a0f96475f 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -1,463 +1,61 @@ -# Rootless Podman Networking +# Podman Networking -Deep-dive into how networking works in the Podman compute driver when running -rootless with pasta as the network backend. Covers the external tooling -(Podman, Netavark, pasta, aardvark-dns), the three nested namespace layers, and -the complete data paths for SSH, outbound traffic, and supervisor-to-gateway -communication. - -For the general Podman driver architecture, lifecycle, API surface, and driver -comparison, see [README.md](README.md). - -## Component Stack - -Podman's networking is composed of four independent projects: - -| Component | Language | Role | -|---|---|---| -| Podman | Go | Container runtime; orchestrates network lifecycle. | -| Netavark | Rust | Network backend; creates interfaces, bridges, firewall rules. | -| aardvark-dns | Rust | Authoritative DNS server for container name resolution. | -| pasta, part of passt | C | User-mode networking; L2-to-L4 socket translation for rootless containers. | - -The key split: rootful containers default to Netavark bridge networking with -real kernel interfaces, while rootless containers commonly use pasta user-mode -networking without needing host privileges. - -## How Netavark Works - -Netavark is invoked by Podman as an external binary. It reads a JSON network -configuration from STDIN and executes one of three commands: - -- `netavark setup ` creates interfaces, assigns IPs, and sets up - firewall rules for NAT and port-forwarding. -- `netavark teardown ` reverses setup and removes interfaces and - firewall rules. -- `netavark create` takes a partial network config and completes it by - assigning subnets and gateways. - -For rootful bridge networking: - -1. Podman creates a network namespace for the container. -2. Podman invokes `netavark setup` with the network config JSON. -3. Netavark creates a bridge, such as `podman0`, if it does not exist. The - default subnet is `10.88.0.0/16`. -4. Netavark creates a veth pair. One end goes into the container's netns and - the other attaches to the bridge. -5. Netavark assigns an IP from the subnet to the container's veth interface. -6. Netavark configures iptables or nftables rules for masquerade and port - mappings. -7. Netavark starts aardvark-dns when DNS is enabled, listening on the bridge - gateway address. - -```text -Host Kernel - | - +-- Bridge interface, such as "podman0" - | | - | +-- veth pair endpoint, host side, container 1 - | +-- veth pair endpoint, host side, container 2 - | - +-- Host physical interface, such as eth0 - | - +-- NAT, iptables or nftables rules managed by Netavark -``` - -Netavark also supports macvlan networks, where the container gets a -sub-interface of a physical host NIC with its own MAC address, and external -plugins via a documented JSON API. - -## How Pasta Works - -Unprivileged users cannot create network interfaces on the host. They cannot -create veth pairs, bridges, or iptables rules. Netavark's bridge approach -cannot work directly for rootless containers without an additional rootless -networking layer. - -Pasta, part of the `passt` project, operates in userspace and translates -between the container's L2 TAP interface and the host's L4 sockets. It requires -no capabilities or privileges. - -```text -Container Network Namespace - | - +-- TAP device, such as "eth0" - | ^ - | | L2 frames, Ethernet - | v - +-- pasta process, userspace - | - | Translation: L2 frames <-> L4 sockets - | - v - Host Network Stack, native TCP/UDP/ICMP sockets -``` - -For an outbound TCP connection from a container: - -1. The application calls `connect()` to an external address. -2. The kernel routes the packet through the default gateway to the TAP device. -3. Pasta reads the raw Ethernet frame from the TAP file descriptor. -4. Pasta parses L2/L3/L4 headers and identifies the TCP SYN. -5. Pasta opens a native TCP socket on the host and calls `connect()` to the - same destination. -6. When the host socket connects, pasta reflects the SYN-ACK back through the - TAP as an L2 frame. -7. For ongoing data transfer, pasta translates between TAP frames and the host - socket, coordinating TCP windows and acknowledgments between the two sides. - -Pasta does not maintain per-connection packet buffers. It reflects observed -sending windows and ACKs directly between peers. This is a thinner translation -layer than a full TCP/IP stack. - -### Built-in Services - -Pasta includes minimal network services so the container stack can -auto-configure: - -| Service | Purpose | -|---|---| -| ARP proxy | Resolves the gateway address to the host's MAC address. | -| DHCP server | Hands out a single IPv4 address, usually matching the host's upstream interface. | -| NDP proxy | Handles IPv6 neighbor discovery and SLAAC prefix advertisement. | -| DHCPv6 server | Hands out a single IPv6 address, usually matching the host's upstream interface. | - -By default there is no NAT. Pasta copies the host's IP addresses into the -container namespace. - -### Local Connection Bypass - -For connections between the container and the host, pasta implements a local -bypass path: - -- Packets with a local destination skip L2 translation. -- TCP uses `splice(2)`. -- UDP uses `recvmmsg(2)` and `sendmmsg(2)`. - -### Port Forwarding - -By default, pasta uses auto-detection. It scans `/proc/net/tcp` and -`/proc/net/tcp6` periodically and automatically forwards ports that are bound -and listening. Port forwarding is configurable through pasta options. - -### Security Properties - -Pasta is designed for rootless use: - -- No dynamic memory allocation after startup. -- All capabilities dropped, except `CAP_NET_BIND_SERVICE` when granted. -- Restrictive seccomp profile. -- Detaches into its own user, mount, IPC, UTS, and PID namespaces. -- No external dependencies beyond libc. - -### Inter-Container Limitation - -Unlike bridge networking, pasta containers are isolated from each other by -default. No virtual bridge connects them. Communication requires port mappings -through the host, pods with a shared network namespace, or opting into rootless -Netavark bridge networking with `podman network create`. - -## Three Nested Namespaces - -The Podman compute driver creates three layers of network isolation: +Only the external supervisor has external network connectivity. The workload +container uses `network=none`; its loopback DNS relay and TCP socket mediation +reach the supervisor through a protected Unix socket, not a veth or proxy +environment variable. ```text -Namespace 1: Host - | - pasta manages port forwarding, such as 127.0.0.1: - gateway listens on its configured bind address and port - | -Namespace 2: Rootless Podman network namespace, managed by pasta - | - Bridge "openshell", often 10.89.x.0/24 - aardvark-dns for container name resolution - | - Container netns - supervisor, proxy, and relay client run here - | -Namespace 3: Inner sandbox netns, created by supervisor - | - veth pair, such as 10.200.0.1 <-> 10.200.0.2 - nftables forces ordinary traffic through proxy - user workload runs here +workload container supervisor container +agent -> sandbox -- private UDS / gRPC -> policy proxy -> Podman network -> destination + | + +-- authenticated gateway callback ``` -Pasta bridges namespace 1 and 2. The veth pair bridges namespace 2 and 3. The -proxy at the boundary of namespace 2 and 3 enforces network policy. +## Outer network fence -### Layer 1 Pasta +The driver creates the workload without networks, host aliases, published +ports, or added capabilities. It checks the Podman inspect response before +launch and restart. The sandbox installs seccomp mediation and Landlock before +executing the agent. It does not create network namespaces, configure nftables, +or require `CAP_NET_ADMIN`. -At driver startup, the driver ensures a Podman bridge network exists: +TCP opens, TCP byte streams, DNS requests/replies, and lifecycle operations +share the authenticated gRPC channel. DNS is resolved and authorized by the +supervisor. General UDP is unsupported. -```rust -client.ensure_network(&config.network_name).await?; -``` +## Supervisor callback network -This creates a bridge network named `openshell` by default, with DNS enabled. -In rootless mode, this bridge can exist inside a user namespace managed by -pasta. The bridge IP range is not reliably routable from the host. - -```text -Host - | - 127.0.0.1:, pasta binds this on the host - | - pasta process, translates L4 sockets <-> L2 TAP frames - | - rootless network namespace - | - Bridge "openshell", such as 10.89.1.0/24 - | - +-- 10.89.1.1, bridge gateway and aardvark-dns - | - +-- veth to container netns - | - 10.89.1.2, container IP -``` - -### Layer 2 Container Networking - -The container spec configures: - -- `nsmode: "bridge"` to use the Podman bridge network. -- `networks` to attach to the configured bridge, `openshell` by default. -- `portmappings` with `host_port: 0`, `container_port: 2222`, and `protocol: - "tcp"` to publish the SSH compatibility port on an ephemeral host port. -- `hostadd` entries for `host.containers.internal` and - `host.openshell.internal`, using Podman's `host-gateway` resolver or the - configured `host_gateway_ip`. - -Pasta is not explicitly configured by the driver. The driver requests bridge -mode and logs the network backend that Podman reports at startup. - -The `host.containers.internal` hostname is injected into `/etc/hosts` so the -supervisor can reach the gateway on the host. Linux defaults to -`host-gateway`; macOS Podman machine defaults to `192.168.127.254`, gvproxy's -host-loopback IP, because older Podman machine images can fail to resolve -`host-gateway`. Override this with `host_gateway_ip` or -`OPENSHELL_PODMAN_HOST_GATEWAY_IP` when a Podman machine uses a non-standard -host-loopback address. - -If `OPENSHELL_GRPC_ENDPOINT` is empty, the driver auto-detects: - -```rust -if config.grpc_endpoint.is_empty() { - let scheme = if config.tls_enabled() { - "https" - } else { - "http" - }; - config.grpc_endpoint = - format!("{scheme}://host.containers.internal:{}", config.gateway_port); -} -``` - -The bridge gateway IP is not a stable substitute in rootless mode because it -can live inside the user namespace rather than on the host. - -Before the gateway binds its serving sockets, the driver reports the callback -listener required by the selected topology: - -- Rootful Linux Podman reports the configured bridge's gateway address exactly. -- Rootless Linux Podman explicitly reporting pasta requests the private IPv4 - source address selected by the host's default route. This avoids guessing - among private interfaces on a multihomed host. -- Rootless Linux Podman reporting slirp4netns, another named helper, or no - helper cannot use a direct local callback listener. The driver fails startup - unless `grpc_endpoint` names an explicitly remote endpoint. Supporting - slirp4netns requires a relay inside Podman's rootless network namespace. -- Podman Machine requests IPv4 loopback because gvproxy terminates the host - forwarding path there. -- An explicitly remote callback endpoint requests no additional local listener. - -On Linux, an explicit `host_gateway_ip` is reported exactly for rootful Podman -and rootless pasta because the driver maps both local callback aliases to that -literal. Other rootless helpers still fail closed. Podman Machine requests -gateway loopback because its configured address is guest-visible and gvproxy -terminates that route on host loopback. The gateway validates and binds every -accepted callback requirement. If the primary listener covers the requested -address, the gateway reuses it and relies on sandbox JWT authorization to limit -the supervisor's RPCs. Otherwise, it creates an additional listener that -exposes only the gateway's sandbox-callable gRPC methods. Operator, health, -reflection, and HTTP requests must use the primary listener. - -### Layer 3 Inner Sandbox Network Namespace - -Inside the container, the supervisor creates another network namespace for the -user workload: - -```text -Container on the Podman bridge - | - Supervisor process, running in container's default netns - | - +-- Proxy listener at the inner namespace gateway address - | - +-- veth pair - | - +-- Inner network namespace - | - sandbox-side veth address - | - default route -> supervisor-side veth address - | - user code runs here - | - nftables rules: - ACCEPT -> proxy TCP - ACCEPT -> loopback - ACCEPT -> established/related - LOG -> TCP SYN bypass attempts - REJECT -> TCP - LOG -> UDP bypass attempts - REJECT -> UDP -``` - -The supervisor uses `nsenter --net=` rather than `ip netns exec` to avoid sysfs -remount issues that arise under rootless Podman where real host -`CAP_SYS_ADMIN` is unavailable. - -For a policy with explicit `protocol: tcp` endpoints, this same inner namespace -also hosts policy DNS and transparent TCP capture. The supervisor answers only -policy-eligible names with epoch-scoped synthetic addresses, redirects TCP to -those synthetic ranges into its transparent listener, and leaves direct real-IP -dials subject to the terminal bypass fence. The Podman driver advertises this -substrate through its driver-owned runtime capability; sandbox image and policy -environment values cannot opt into it independently. - -The container spec preserves Podman's resolver search domains and options. -Policy DNS captures both UDP and TCP in the inner namespace, so it does not -depend on libc honoring `use-vc` and does not change ordinary short-name -resolution for sandboxes that do not use native TCP. - -A tmpfs is mounted at `/run/netns` in the container spec so the supervisor can -create named network namespaces. In rootless Podman this directory does not -exist on the host, so a private tmpfs gives the supervisor its own writable -`/run/netns` without needing host filesystem access. - -## Complete Data Paths - -### SSH Session - -```text -Client, openshell CLI - | - 1. gRPC: CreateSshSession -> gateway, returns token and connect_path - 2. HTTP CONNECT /connect/ssh to gateway - headers: x-sandbox-id, x-sandbox-token - | -Gateway - | - 3. Looks up SupervisorSession for sandbox_id - 4. Sends RelayOpen{channel_id} over ConnectSupervisor bidi stream - | - gRPC traverses host -> pasta translation -> container bridge - | -Supervisor inside container - | - 5. Receives RelayOpen, opens new RelayStream RPC back to gateway - 6. Sends RelayInit{channel_id} on the stream - 7. Connects to Unix socket /run/openshell/ssh.sock - 8. Bidirectional bridge: RelayStream <-> Unix socket - | -SSH daemon inside container, Unix socket only - | - 9. Authenticates. Access is gated by the relay chain. - 10. Spawns shell process - 11. Shell enters inner netns via setns(fd, CLONE_NEWNET) - | -User shell in sandbox netns -``` - -The SSH daemon listens on a Unix socket with restrictive permissions. The -published TCP port mapping exists in the container spec for compatibility and -health/debug paths. Normal SSH communication uses the gRPC reverse-connect relay -pattern. - -### Outbound HTTP Request - -```text -User code in inner netns - | - 1. curl https://api.example.com - HTTP_PROXY points at the local sandbox proxy - | - 2. TCP connect to proxy - allowed by nftables as the only ordinary egress destination - | - 3. HTTP CONNECT api.example.com:443 - | -Supervisor proxy in container netns - | - 4. Policy evaluation with process identity - 5. SSRF check - 6. Optional L7 TLS intercept and HTTP method/path inspection - | - 7. If allowed, TCP connect to api.example.com:443 - from the container netns - | - 8. Through Podman bridge -> pasta -> host -> internet -``` - -### Supervisor gRPC Callback - -The Podman driver auto-detects the callback endpoint scheme based on whether -TLS client certificates are configured. When the RPM's auto-generated PKI is in -place, the endpoint is `https://host.containers.internal:17670` and the -supervisor connects with mTLS. Without TLS configuration, it falls back to -`http://host.containers.internal:`. - -```text -Supervisor in container netns - | - 1. Connects to host.containers.internal: - with mTLS when OPENSHELL_TLS_* paths are set - | - 2. Routed through container default gateway - | - 3. Pasta translates L2 frame -> host L4 socket when rootless backend uses pasta - | - 4. Host TCP socket connects to gateway - | -Gateway - | - 5. TLS handshake when enabled - 6. ConnectSupervisor bidirectional stream established - 7. Heartbeats at the interval accepted by the gateway - 8. Reconnects with exponential backoff on failure - 9. Same gRPC channel reused for RelayStream calls -``` +The configured `network_name`, host-gateway aliases, upstream corporate proxy, +and published SSH port apply only to the supervisor companion. The gateway's +SSH tunnel still uses the supervisor relay, not the published port. -The gateway binds to `127.0.0.1:17670` by default in the RPM packaging. Client -certificates are auto-generated by `openshell-gateway generate-certs` on first -start and bind-mounted into sandbox containers by the Podman driver. +Rootful Podman uses the configured bridge and its gateway address. Rootless +local callbacks require the existing pasta path; slirp4netns or unknown helpers +require an explicitly remote `grpc_endpoint`. On macOS, Podman Machine provides +the runtime and host-loopback forwarding. -## Differences from the Kubernetes Driver +These runtime-managed network helpers are outside the workload trust boundary. +Sharing the workload's user namespace preserves volume UID/GID mapping; it +does not share the workload's PID, mount, or network namespaces. -| Aspect | Kubernetes | Podman, rootless pasta | -|---|---|---| -| Container or pod IP | Routable cluster-wide | Non-routable from the host in common rootless setups. | -| Network reachability | Pod IPs reachable from gateway | Bridge not reliably routable from host; requires host aliases or published ports. | -| Sandbox to gateway | Direct TCP to Kubernetes service or endpoint | `host.containers.internal` through bridge and rootless backend. | -| SSH transport | Reverse gRPC relay | Reverse gRPC relay. | -| Port publishing | Not needed for relay | Ephemeral host port remains in the container spec for compatibility and debug paths. | -| TLS | mTLS via Kubernetes secrets | mTLS via mounted client files, RPM defaults, or explicit configuration. | -| DNS | Kubernetes CoreDNS | Podman bridge DNS through aardvark-dns when DNS is enabled. | -| Network policy | Kubernetes network policy for pod ingress plus supervisor policy | nftables inside inner sandbox netns plus supervisor policy. | -| Supervisor delivery | Kubernetes driver managed pod image or template | OCI image volume mount. | -| Secrets | Kubernetes Secret volume and env vars | Per-sandbox JWT via Podman secret; TLS client materials from configured host files. | +## Troubleshooting -Both drivers use the same reverse gRPC relay for SSH transport. The most -important Podman-specific difference is network reachability: in rootless -Podman, the bridge network is not reliably routable from the host, so -host-to-container and container-to-host communication must use host aliases, -published ports, or the supervisor relay. +Inspect both containers with the same sandbox-ID label, distinguishing +`openshell.io/isolation-role=sandbox` from +`openshell.io/isolation-role=supervisor`. -## Port Assignments +- Sandbox fails its qualification probe: use its log to identify the denied + kernel/runtime primitive. Do not add capabilities or disable runtime seccomp. +- Sandbox cannot authenticate to supervisor: check the private channel volume, + matching user namespace mappings, and shared SELinux label. +- Supervisor cannot call back: inspect its configured gateway endpoint, + credentials, Podman network, and gateway callback listener. +- DNS or egress denied: inspect supervisor policy decisions. Do not add a + workload network, resolver bypass, or direct gateway route. +- Pair is not Ready: check the supervisor health socket and gateway session. + A running workload container alone does not establish readiness. -| Port | Component | Purpose | -|---|---|---| -| `17670` | Gateway | Default local gRPC and HTTP multiplexed server port. | -| `2222` | Sandbox | Container port mapping default for the SSH compatibility port. | -| `3128` | Sandbox proxy | HTTP CONNECT proxy inside the sandbox network model. | -| `0` | Host | Ephemeral host port requested for the container SSH compatibility port. | +See the [driver overview](README.md) and +[Podman runtime documentation](https://docs.podman.io/en/latest/markdown/podman-run.1.html) +for runtime options. diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 5781594158..17bacce1ae 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -1,489 +1,112 @@ # openshell-driver-podman -The Podman compute driver manages sandbox containers via the Podman REST API -over a Unix socket. It targets single-machine and developer environments where -rootless container isolation is preferred over a full Kubernetes cluster. The -driver runs in-process within the gateway server and delegates all sandbox -isolation enforcement to the `openshell-sandbox` supervisor binary, which is -sideloaded into each container via an OCI image volume mount. +The Podman compute driver runs inside the gateway and uses the native libpod +REST API over a Unix socket. Each sandbox has two independent containers: -When the gateway configures `[openshell.gateway.otlp]`, Podman compute-driver -spans export to the same OTLP/gRPC collector with the service name -`openshell-driver-podman`. The driver preserves the gateway trace context and -uses the same compute-driver RPC span names in its in-process and standalone -forms. +- `openshell-sandbox` owns the agent process in the workload container. +- `openshell-supervisor` evaluates policy, holds gateway credentials, and + proxies approved egress in a separate companion container. -`mise run gateway:podman` enables this export only when a local collector is -listening on `127.0.0.1:4317`. Otherwise, it omits the gateway OTLP configuration -so the development gateway does not repeatedly report export failures. +The driver provisions placement, identity, credentials, transport, and lifecycle. +The shared isolation interface supplies exec, attach, signal, terminate, binary +identity, DNS, TCP, and loopback-forwarding semantics. -Before creating the container, the driver inspects the final sandbox image and -captures its immutable image ID and raw OCI `Config.User`. Container creation -uses that image ID with pulling disabled, preventing a mutable tag from changing -between inspection and launch. The supervisor runs as root, resolves omitted -policy identity fields from the image declaration, and drops only agent -children to the completed identity. Named OCI components remain names after -validation; a missing group is filled with the user's numeric primary GID. Explicit -`process.run_as_user` and `process.run_as_group` values take precedence -independently. +## Runtime posture -For a rootless networking deep dive, see [NETWORKING.md](NETWORKING.md). - -## Stop and Start - -Stop stops the managed container without deleting it. The per-sandbox named -workspace volume, token and proxy-auth secrets, labels, and container metadata -remain intact. Start starts the same container and reuses the same named -volume. Stopped managed containers remain visible through list and watch -reconciliation. Delete remains responsible for removing the container, -driver-owned secrets, and workspace volume. - -The stop call waits until Podman reports the container as stopped or exited. -This keeps an immediate start from racing a rootless Podman stop that is still -finishing after its API request returns. - -Graceful gateway shutdown sends `StopSandbox` for each sandbox whose persisted -phase requires running compute without changing that persisted intent. On -startup, the gateway sends an idempotent `StartSandbox` request for the same -sandboxes, restarting their retained containers. Explicitly stopped sandboxes -remain excluded. - -## Architecture - -The Podman driver communicates with the Podman daemon over a Unix socket and -delegates sandbox isolation to the supervisor binary running inside each -container. - -```mermaid -graph TB - CLI["openshell CLI"] -->|gRPC| GW["Gateway Server
(openshell-server)"] - GW -->|in-process| PD["PodmanComputeDriver"] - PD -->|HTTP/1.1
Unix socket| PA["Podman API"] - PA -->|OCI runtime
crun/runc| C["Sandbox Container"] - C -->|image volume
read-only| SV["Supervisor Binary
/opt/openshell/bin/openshell-sandbox"] - SV -->|creates| NS["Nested Network Namespace
veth pair + proxy"] - SV -->|enforces| LL["Landlock + seccomp"] - SV -->|gRPC callback| GW -``` - -## Isolation Model - -The Podman driver provides the same protection layers as the other compute -drivers. The driver itself does not implement isolation primitives directly. It -configures the container so that the `openshell-sandbox` supervisor can enforce -them at runtime. - -### Container Security Configuration - -The container spec in `container.rs` sets these security-critical fields: - -| Setting | Value | Rationale | +| Property | Workload | Supervisor | |---|---|---| -| `user` | `0:0` | The supervisor needs root inside the container for namespace creation, proxy setup, Landlock, seccomp, and filesystem preparation. | -| `cap_drop` | Selected unneeded defaults | Podman's default capability set is already restricted. The driver drops capabilities the supervisor does not need. | -| `cap_add` | `SYS_ADMIN`, `NET_ADMIN`, `SYS_PTRACE`, `SYSLOG`, `DAC_READ_SEARCH`, `SETPCAP`, `KILL` | Grants supervisor-only capabilities required for namespace setup, process identity, bypass diagnostics, child bounding-set cleanup, and forwarding shutdown signals to a workload that runs as the sandbox user. Policy DNS binds an unprivileged supervisor port and does not require `NET_BIND_SERVICE`. | -| `no_new_privileges` | `true` | Prevents privilege escalation after exec. | -| `seccomp_profile_path` | `unconfined` | The supervisor installs its own policy-aware BPF filter. A container-level profile can block Landlock/seccomp syscalls during setup. | -| `mounts` | Private tmpfs at `/run/netns` | Lets the supervisor create named network namespaces in rootless Podman. | -| CDI GPU devices | Opaque `driver_config.cdi_devices` values when set, otherwise the requested count of NVIDIA CDI GPUs selected in round-robin order. Local `/dev/dxg` permits `nvidia.com/gpu=all` as a WSL2 all-only compatibility fallback, where it counts as one selectable device. | Exposes requested GPUs to GPU-enabled sandbox containers. Exact CDI device lists must not contain duplicates and must match the effective GPU count. | - -The restricted agent child does not retain these supervisor privileges. - -## Driver Config Mounts - -The gateway forwards the `podman` block from `--driver-config-json` to this -driver. The driver accepts user-supplied `mounts` entries with these Podman -mount types: - -- `bind`: mounts an absolute host path when `[openshell.drivers.podman]` - has `enable_bind_mounts = true`. -- `volume`: mounts an existing Podman named volume. The driver validates that - the volume exists before provisioning and never creates or removes it. Podman - local-driver volumes created with bind options are treated as host bind - mounts and require `enable_bind_mounts = true`. -- `tmpfs`: mounts an in-memory filesystem with optional `options`, - `size_bytes`, and `mode`. -- `image`: mounts an OCI image through Podman's image-volume API. The driver - pulls the image during provisioning using the sandbox image pull policy. - -Host bind mounts are disabled by default because they expose gateway host paths -to sandbox requests. The driver still uses internal bind mounts for configured -TLS material; per-sandbox gateway JWTs are delivered through Podman secrets. - -Podman `bind` mounts accept `source`, `target`, optional `read_only`, and an -optional `selinux_label` of `shared` (applies `:z`) or `private` (applies -`:Z`) for SELinux-enforcing hosts. User-supplied bind and volume mounts are -read-only by default; set `read_only: false` to make them writable. Podman -image and volume mounts do not support `subpath` in OpenShell driver config. -Mount `source` and `target` values must not contain surrounding whitespace. -Mount targets must be absolute container paths and must not replace -the workspace root (`/sandbox`) or overlap OpenShell supervisor files, -`/etc/openshell`, `/etc/openshell-tls`, or `/run/netns`. - -Example named-volume usage: - -```shell -podman volume create openshell-work - -openshell sandbox create \ - --driver-config-json '{"podman":{"mounts":[{"type":"volume","source":"openshell-work","target":"/sandbox/work"}]}}' \ - -- claude -``` - -### Capability Breakdown - -| Capability | Purpose | -|---|---| -| `SYS_ADMIN` | seccomp filter installation, namespace creation, and Landlock setup. | -| `NET_ADMIN` | Network namespace veth setup, IP address assignment, routes, and nftables. | -| `SYS_PTRACE` | Reading `/proc//exe` and walking process ancestry for binary identity. | -| `SYSLOG` | Reading `/dev/kmsg` for bypass-detection diagnostics. | -| `DAC_READ_SEARCH` | Reading `/proc//fd/` across UIDs so the proxy can resolve the binary responsible for a connection. | -| `SETPCAP` | Clearing the restricted child process capability bounding set before exec. | - -The driver intentionally keeps Podman's default `SETUID`, `SETGID`, `CHOWN`, -and `FOWNER` capabilities because the supervisor needs them to drop privileges -and prepare writable sandbox directories. It also keeps `SETPCAP` until child -setup so `drop_privileges()` can clear the child capability bounding set before -exec. It drops unneeded defaults such as -`DAC_OVERRIDE`, `FSETID`, `KILL`, `NET_RAW`, `SETFCAP`, -and `SYS_CHROOT`. - -## Supervisor Sideloading - -The supervisor binary is delivered to sandbox containers via Podman's OCI image -volume mechanism, distinct from both the Kubernetes pod-volume approach and the -VM's embedded guest bundle. - -```mermaid -sequenceDiagram - participant D as PodmanComputeDriver - participant P as Podman API - participant C as Sandbox Container - - D->>P: pull_image(supervisor, "missing") - D->>P: create_container(spec with image_volumes) - Note over P: Podman resolves image_volumes at
libpod layer before OCI spec generation - P->>C: Mount supervisor image at /opt/openshell/bin (read-only) - D->>P: start_container - C->>C: entrypoint: /opt/openshell/bin/openshell-sandbox -``` - -The supervisor image from `deploy/docker/Dockerfile.supervisor` provides the -static `openshell-sandbox` binary at `/openshell-sandbox`. -Mounting that image at `/opt/openshell/bin` makes the binary available as -`/opt/openshell/bin/openshell-sandbox`. - -The container spec sets that binary as the entrypoint. This avoids relying on -the sandbox image entrypoint or command, which might otherwise append the -supervisor path as an argument to an image-provided shell. - -## TLS - -When all three Podman TLS paths are set, the driver treats sandbox callbacks as -mTLS callbacks: - -- `OPENSHELL_PODMAN_TLS_CA` -- `OPENSHELL_PODMAN_TLS_CERT` -- `OPENSHELL_PODMAN_TLS_KEY` - -The driver validates that the TLS paths are provided as a complete set. Partial -configuration fails early instead of silently falling back to plaintext. - -When enabled, the driver: - -1. Switches the auto-detected endpoint scheme from `http://` to `https://`. -2. Bind-mounts the client cert files read-only into the container at - `/etc/openshell/tls/client/`. -3. Sets `OPENSHELL_TLS_CA`, `OPENSHELL_TLS_CERT`, and `OPENSHELL_TLS_KEY` to - the container-side paths. - -The supervisor reads these env vars and uses them to establish an mTLS -connection back to the gateway. On SELinux systems, the bind mounts include -Podman's shared relabel option so the container process can read the files. - -The RPM packaging auto-generates a self-signed PKI on first start via -`openshell-gateway generate-certs`. Client certs are placed in the CLI -auto-discovery directory (`~/.config/openshell/gateways/openshell/mtls/`) so -the CLI connects with mTLS without manual configuration. See -`deploy/rpm/CONFIGURATION.md` for the full RPM configuration reference. - -## Network Model - -Sandbox network isolation uses a two-layer approach: a Podman bridge network -for container-to-host communication, and a nested network namespace created by -the supervisor for sandbox process isolation. - -```mermaid -graph TB - subgraph Host - GW["Gateway Server
127.0.0.1:17670"] - PS["Podman Socket"] - end - - subgraph Bridge["Podman Bridge Network (10.89.x.x)"] - subgraph Container["Sandbox Container"] - SV["Supervisor
(root in user ns)"] - subgraph NestedNS["Nested Network Namespace"] - SP["Sandbox Process
(resolved non-root identity)"] - VE2["veth1: 10.200.0.2"] - end - VE1["veth0: 10.200.0.1
(CONNECT proxy)"] - SV --- VE1 - VE1 ---|veth pair| VE2 - end - end - - GW -.->|SSH via supervisor relay
gRPC session| SV - SV -->|gRPC callback via
host.containers.internal| GW - SP -->|all egress via proxy| VE1 +| UID/GID | Pinned non-root workload identity | Same mapped identity | +| Capabilities | Drop all; add none | Drop all; add none | +| Seccomp | Runtime default plus sandbox-installed filters | Runtime default | +| Network | `none`; loopback only | Configured Podman network | +| Gateway JWT and upstream credentials | Never mounted | Podman secrets | +| User volumes and CDI devices | Workload only | Never mounted | +| Channel | Private named volume, writable | Same volume, read-only | + +Podman creates the namespaces and volume ownership before the workload runs. +Rootless operation uses the operator's Podman service and subordinate-ID +configuration; it does not require adding capabilities to either container. +The supervisor joins the workload's **user namespace only** to preserve UID/GID +mapping for shared-volume access. PID, mount, and network namespaces remain +separate. The channel volume uses shared SELinux relabeling (`:z`). + +The runtime must pass the sandbox's unprivileged enforcement probe, including +nested seccomp notification and Landlock. Unsupported runtime defaults fail +closed; do not switch to an unconfined profile or add capabilities. + +## Protected channel and network enforcement + +```text +agent -> openshell-sandbox === authenticated gRPC / private UDS === supervisor -> network + network=none TCP, DNS, control streams policy + gateway JWT ``` -Key points: - -- Bridge network: created by `client.ensure_network()` with DNS enabled. - Containers on the bridge can see each other at L3, but sandbox processes - cannot because they are isolated inside the nested netns. -- Nested netns: the supervisor creates a private `NetworkNamespace` with a veth - pair. Sandbox processes enter this netns via `setns(fd, CLONE_NEWNET)` in the - `pre_exec` hook, forcing ordinary traffic through the CONNECT proxy. -- Policy DNS and transparent TCP: the driver advertises the complete - `policy-dns-transparent-tcp` substrate. For explicit `protocol: tcp` - endpoints, the supervisor installs namespace-local DNS listeners, synthetic - routes, and TCP redirect rules before starting the workload. The container - disables Podman's implicit DNS search suffix so policy DNS evaluates the - exact endpoint name requested by the workload, and asks libc to use the - policy DNS TCP listener to avoid rootless Podman's nested UDP NAT return - path. -- Port publishing: the container spec still requests `host_port: 0` for the - configured SSH port. The gateway SSH tunnel uses the supervisor relay rather - than connecting directly to the published port. -- Host gateway: `host.containers.internal` and `host.openshell.internal` are - injected into `/etc/hosts` so containers can reach services on the gateway - host. Linux defaults to Podman's `host-gateway` resolver. macOS Podman - machine defaults to gvproxy's host-loopback IP, `192.168.127.254`, because - stale Podman machines may fail to resolve `host-gateway`. -- nsenter: the supervisor uses `nsenter --net=` instead of `ip netns exec` for - namespace operations, avoiding the sysfs remount path that fails in rootless - containers. - -See [NETWORKING.md](NETWORKING.md) for the rootless Podman networking deep dive. - -## Supervisor Relay - -Podman follows the same end-to-end contract as the Kubernetes and VM drivers -for the in-container SSH relay: gateway config to `PodmanComputeConfig` to -sandbox environment to supervisor session registration on that path. - -1. `[openshell.drivers.podman].ssh_socket_path` is deserialized into - `PodmanComputeConfig::ssh_socket_path` when the gateway builds the in-process - driver. The field defaults to `/run/openshell/ssh.sock` when omitted. -2. `build_env()` in `container.rs` sets `OPENSHELL_SSH_SOCKET_PATH` to that - value, alongside required vars such as `OPENSHELL_ENDPOINT` and - `OPENSHELL_SANDBOX_ID`. These driver-controlled entries overwrite template - environment variables to prevent spoofing. -3. The supervisor reads `OPENSHELL_SSH_SOCKET_PATH` and uses it for the Unix - socket the gateway's SSH stack bridges to. - -The standalone `openshell-driver-podman` binary sets the same struct field from -`OPENSHELL_SANDBOX_SSH_SOCKET_PATH`. - -## Credential Injection - -Sandboxes authenticate to the gateway via mTLS using client materials bind- -mounted into the container from a Podman secret. No shared per-request secret -is injected as an environment variable. - -| Credential | Mechanism | Visible in `inspect`? | Visible in `/proc//environ`? | -|---|---|---|---| -| mTLS client cert/key | Bind-mounted file paths (`OPENSHELL_TLS_*` env vars point at them) | Yes (paths only) | Yes (paths only) | -| Sandbox identity | Plaintext env var | Yes | Yes | -| gRPC endpoint | Plaintext env var, override-protected | Yes | Yes | -| Supervisor relay socket path | Plaintext env var, override-protected | Yes | Yes | - -The `build_env()` function inserts user-supplied variables first, then -unconditionally overwrites all security-critical variables to prevent spoofing -via sandbox templates: - -- `OPENSHELL_SANDBOX` -- `OPENSHELL_SANDBOX_ID` -- `OPENSHELL_ENDPOINT` -- `OPENSHELL_SSH_SOCKET_PATH` -- `OPENSHELL_CONTAINER_IMAGE` -- `OPENSHELL_MAIN_PROCESS_SPEC` - -## Sandbox Lifecycle - -### Creation Flow - -```mermaid -sequenceDiagram - participant GW as Gateway - participant D as PodmanComputeDriver - participant P as Podman API - - GW->>D: create_sandbox(DriverSandbox) - D->>D: validate name + id - D->>D: validated_container_name() - - D->>P: pull_image(supervisor, "missing") - D->>P: pull_image(sandbox_image, policy) - - D->>P: create_volume(workspace) - Note over D: On failure below, rollback volume - - D->>P: create_container(spec) - alt Conflict (409) - D->>P: remove_volume - D-->>GW: AlreadyExists - end - Note over D: On failure below, rollback container + volume - - D->>P: start_container - D-->>GW: Ok -``` - -Each step rolls back previously-created resources on failure. The Conflict path -cleans up the volume because it is keyed by the new sandbox's ID, not the -conflicting container's ID. - -### Readiness and Health - -The container `healthconfig` marks the sandbox healthy when any of these -signals succeeds: - -- Legacy marker file `/var/run/openshell-ssh-ready`. -- `test -S` on the configured supervisor Unix socket path. -- The prior TCP check for a listener on the in-container SSH port. - -The Unix socket check allows relay-only backend readiness when the supervisor -exposes the socket without the old marker or published-port signal. Omitting -`health_check_interval_secs` disables these Podman/conmon probes, but it does -not bypass public readiness gating: the gateway keeps a backend-ready sandbox -in `Provisioning` with `SupervisorNotConnected` until its supervisor control -session is connected. - -### Deletion Flow - -1. Validate `sandbox_name` and stable `sandbox_id` from `DeleteSandboxRequest`. -2. Best-effort inspect cross-checks the container label when present, but - cleanup remains keyed by the request `sandbox_id`. -3. Best-effort stop, ignoring the stop result. -4. Force-remove the container. -5. Remove workspace volume derived from the request `sandbox_id`, warning on - failure and continuing. - -If the container is already gone during inspect or remove, the driver still -performs idempotent volume cleanup using the request `sandbox_id` and -returns `Ok(false)` for the container-delete result. This prevents leaked -Podman resources after out-of-band container removal or label drift. - -## Configuration - -| Environment Variable | CLI Flag | Default | Description | -|---|---|---|---| -| `OPENSHELL_PODMAN_SOCKET` | `--podman-socket` | Probes known local Podman API sockets and uses the first responsive socket, then falls back to asking the `podman` CLI for the host-side socket. Fails to start if neither finds one. | Podman API Unix socket path. | -| `OPENSHELL_SANDBOX_IMAGE` | `--sandbox-image` | From gateway config | Default OCI image for sandboxes. | -| `OPENSHELL_SANDBOX_IMAGE_PULL_POLICY` | `--sandbox-image-pull-policy` | `if_not_present` | Pull policy: `always`, `if_not_present`, `never`, or `newer`. | -| `OPENSHELL_GRPC_ENDPOINT` | `--grpc-endpoint` | Auto-detected via `host.containers.internal` | Gateway gRPC endpoint for sandbox callbacks. | -| `OPENSHELL_GATEWAY_PORT` | `--gateway-port` | `17670` | Gateway port used for endpoint auto-detection by the standalone binary. | -| `OPENSHELL_NETWORK_NAME` | `--network-name` | `openshell` | Podman bridge network name. | -| `OPENSHELL_PODMAN_HOST_GATEWAY_IP` | `--host-gateway-ip` | empty on Linux, `192.168.127.254` on macOS | Host gateway IP used for sandbox host aliases. Empty uses Podman's `host-gateway` resolver. | -| `OPENSHELL_SANDBOX_SSH_SOCKET_PATH` | `--sandbox-ssh-socket-path` | `/run/openshell/ssh.sock` | Supervisor Unix socket path in `PodmanComputeConfig`. | -| `OPENSHELL_STOP_TIMEOUT` | `--stop-timeout` | `45` | Container stop timeout in seconds. | -| `OPENSHELL_SANDBOX_PIDS_LIMIT` | `--sandbox-pids-limit` | `2048` | Podman cgroup PID limit for sandbox containers. Omission uses OpenShell's `2048` default; explicit `0` is invalid. | -| `OPENSHELL_SUPERVISOR_IMAGE` | `--supervisor-image` | `ghcr.io/nvidia/openshell/supervisor:latest` through the gateway, required standalone | OCI image containing the supervisor binary. | -| `OPENSHELL_PODMAN_TLS_CA` | `--podman-tls-ca` | unset | Host path to the CA certificate mounted for sandbox mTLS. | -| `OPENSHELL_PODMAN_TLS_CERT` | `--podman-tls-cert` | unset | Host path to the client certificate mounted for sandbox mTLS. | -| `OPENSHELL_PODMAN_TLS_KEY` | `--podman-tls-key` | unset | Host path to the client private key mounted for sandbox mTLS. | -| `OPENSHELL_SANDBOX_HTTPS_PROXY` | `--sandbox-https-proxy` | unset | Corporate forward proxy URL for the supervisor's upstream TLS dials, chained with HTTP CONNECT. Credential-free `http://host:port` and `https://host:port` URLs are supported (scheme and port required). For an `https://` proxy the supervisor TLS-wraps the proxy connection, verifying the proxy certificate against the built-in and system roots plus `--sandbox-proxy-ca-bundle`. Plain-HTTP requests always dial directly. | -| `OPENSHELL_SANDBOX_NO_PROXY` | `--sandbox-no-proxy` | unset | Comma-separated `NO_PROXY` list (hostnames, domain suffixes, IPs, CIDRs, each with an optional `:port` qualifier) dialed directly instead of through the corporate proxy. IP/CIDR entries also match hostnames through their validated DNS resolution. | -| `OPENSHELL_SANDBOX_PROXY_AUTH_FILE` | `--sandbox-proxy-auth-file` | unset | Path to a file containing the proxy credentials as `user:pass`. Staged as a root-only Podman secret so credentials never appear in config or container metadata. Requires the insecure-auth acknowledgement below. | -| `OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE` | `--sandbox-proxy-auth-allow-insecure` | unset | Explicit acknowledgement (`true`) that the credential is sent as cleartext Basic auth over the plain-TCP connection to the `http://` proxy. Required when the auth file is set with an `http://` proxy; not required for `https://` proxies (the credential travels inside the verified TLS session) but tolerated if set. Rejected when no auth file is configured. | -| `OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME` | `--sandbox-proxy-connect-by-hostname` | unset | Send the destination hostname in CONNECT requests instead of a validated IP. Last resort for proxies whose ACLs filter on hostnames: the proxy then resolves the name itself, so sandbox SSRF/`allowed_ips` validation no longer binds the connection. | -| `OPENSHELL_PODMAN_USERNS` | `--userns` | unset | User namespace mode for sandbox containers (e.g. `auto`). When unset, containers use the default user namespace. | -| `OPENSHELL_SANDBOX_PROXY_CA_BUNDLE` | `--sandbox-proxy-ca-bundle` | unset | Path (on the gateway host) to a PEM CA bundle trusted for the corporate proxy. Bind-mounted read-only into the sandbox (a CA certificate is not secret). Trusted for the `https://` proxy TLS handshake and, because TLS-intercepting proxies re-sign tunneled certificates, folded into the sandbox trust bundle and upstream verification. Requires a proxy URL; the file must exist and hold at least one certificate. | - -Through the gateway, the same settings are the `https_proxy`, `no_proxy`, -`proxy_auth_file`, `proxy_auth_allow_insecure`, `proxy_connect_by_hostname`, -and `proxy_ca_bundle` keys under `[openshell.drivers.podman]`; see -`docs/reference/gateway-config.mdx`. - -`provider_spiffe_workload_api_socket` accepts either an absolute host UNIX -Workload API socket, projected through a dedicated read-only mount, or an -explicit container-reachable `tcp:IP:port` endpoint. The driver sets the -supervisor's `OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET` accordingly. -`app_armor_profile` shares the canonical -`RuntimeDefault`, `Unconfined`, or `Localhost/` model with Docker and -Kubernetes. When omitted, the driver sends no override and preserves Podman's -runtime-selected profile. Set `Unconfined` explicitly only when the deployment -requires the supervisor's mount setup to bypass that profile. Explicit confined -choices fail early when Podman reports AppArmor unavailable. - -This is an operator-owned egress boundary: the driver passes the settings on -the supervisor's command line, so sandbox and template environment — and any -`ENV` baked into the sandbox image — cannot override them, and the -conventional `HTTPS_PROXY`/`HTTP_PROXY`/`NO_PROXY` variables a sandbox -controls do not steer it. Credentials must be supplied through -`proxy_auth_file`; an inline `user:pass@` in the URL is rejected at startup. - -Basic auth over an `http://` proxy is cleartext on the wire: anyone on the -network path between the sandbox host and the proxy can recover the -credential. Setting `proxy_auth_file` therefore requires -`proxy_auth_allow_insecure = true`; both the driver and the in-container -supervisor reject credentials without that explicit acknowledgement. - -CONNECT requests target a validated resolved IP by default, so the proxy -performs no DNS resolution and the tunnel stays bound to the address that -passed the sandbox's SSRF and `allowed_ips` checks; the hostname still -travels inside the tunnel (TLS SNI, application `Host`). In split-horizon -networks, point the gateway host at the corporate resolver. Set -`proxy_connect_by_hostname = true` only when the proxy's ACLs filter on -hostnames and reject IP CONNECT targets — it re-opens proxy-side DNS -resolution, making the proxy's ACLs the effective egress control. - -## Rootless-Specific Adaptations - -The Podman driver is designed for rootless operation. The following adaptations -matter compared to cluster or rootful runtimes: - -1. subuid/subgid preflight check: on non-macOS hosts, `check_subuid_range()` in - `driver.rs` warns operators if `/etc/subuid` or `/etc/subgid` entries are - missing for the current user. This is not a hard error because some systems - use LDAP or other mechanisms. macOS skips the check because `podman machine` - runs the Podman service inside a Linux VM. -2. cgroups v2 requirement: the driver refuses to start if cgroups v1 is - detected. Rootless Podman requires the unified cgroup hierarchy. -3. `nsenter` for namespace operations: `openshell-sandbox` uses - `nsenter --net=` instead of `ip netns exec` to avoid the sysfs remount path - that requires real `CAP_SYS_ADMIN` in the host user namespace. -4. `DAC_READ_SEARCH` capability: required for the proxy to read - `/proc//fd/` across UIDs within the user namespace. -5. `SETUID` and `SETGID` capabilities: kept from Podman's default capability - set so `drop_privileges()` can call `setuid()` and `setgid()`. -6. `host.containers.internal`: used instead of Docker's `host.docker.internal` - for container-to-host communication. The driver also injects the - OpenShell-owned `host.openshell.internal` alias. -7. Ephemeral port publishing: the SSH compatibility port uses `host_port: 0` - because the bridge network IP is not reliably routable from the host in - rootless mode. -8. tmpfs at `/run/netns`: a private tmpfs lets the supervisor create named - network namespaces via `ip netns add`. - -## Implementation References - -- Gateway integration: `crates/openshell-gateway/src/lib.rs` registers the - driver factory and constructs `PodmanComputeConfig` from the generic server - build context. -- Server configuration: `crates/openshell-server/src/lib.rs` exposes the - backend-agnostic registry and factory context. -- Gateway relay path: `openshell-core` `Config::sandbox_ssh_socket_path` in - `crates/openshell-core/src/config.rs`. -- SSRF mitigation: `crates/openshell-core/src/net.rs`, - `crates/openshell-sandbox/src/proxy.rs`, and - `crates/openshell-server/src/grpc/policy.rs`. -- Sandbox supervisor: `crates/openshell-sandbox/src/` for Landlock, seccomp, - netns, proxy, and relay behavior shared by all drivers. -- Container engine abstraction: `tasks/scripts/container-engine.sh` for - build/deploy support across Docker and Podman. -- Supervisor image build: `deploy/docker/Dockerfile.supervisor`. +The workload has no external interface or published port. Seccomp socket +mediation carries TCP and DNS through one authenticated gRPC connection. +DNS remains supervisor-mediated; general UDP is unsupported. The driver sets +`net.ipv4.ip_unprivileged_port_start=0` in the isolated workload network +namespace so the sandbox's loopback DNS relay can bind port 53 without a +capability. No nftables or nested network namespace setup runs in the sandbox. + +The channel contains the sandbox bootstrap and sandbox-side TLS identity only. +Supervisor private keys and topology stay in the companion's private filesystem. +Landlock denies agent access to the top-level `/.openshell` control hierarchy. +The driver verifies Podman's reported `network=none` fence before launch and +restart. `host.containers.internal` and callback networking apply to the +supervisor, not the agent. + +Gateway callbacks use the existing sandbox JWT and optional configured mTLS +bundle. The sandbox/supervisor channel always uses its separate, per-sandbox +mutual TLS material. These are distinct authentication relationships. + +## Identity and trusted binaries + +Both workload and supervisor images are pinned by immutable image ID. The +driver reads account files from a stopped workload-image container; it never +executes the image to resolve an account. Policy identity fields override OCI +`USER` independently. Named users/groups resolve against that image, including +supplementary groups. Root and unresolved identities fail before provisioning. +Images must not prepopulate the reserved `/.openshell` hierarchy; this prevents +image-controlled symlinks from aliasing private control state into user mounts. + +The trusted runtime image supplies `/openshell-sandbox` and +`/openshell-supervisor`. Podman's read-only image volume delivers the sandbox +binary; user-namespace modes that cannot use image volumes retain the existing +trusted binary extraction path. Image and request environment belong to agent +children, never the supervisor process. + +## Lifecycle and readiness + +Create builds both stopped containers and stages both private archives before +starting either container. The sandbox does not execute the agent until the +supervisor authenticates and confirms the common boundary contract. Failed +creation removes only containers created by that attempt, then cleans up +driver-owned volumes and secrets. + +Stop retains both containers, workspace, channel, and secrets. Start restores +the consumed sandbox bootstrap from a copy in the supervisor's private +filesystem, verifies the fence, and starts the same pair. A failed supervisor +start stops the workload. Delete removes the companion first, then the workload, +channel, workspace, and driver-owned secrets. User-owned volumes are retained. + +Only workload containers appear in sandbox list/watch results. Readiness uses +the supervisor's private health socket; there is no shell, legacy marker, or +TCP-listener shortcut. Watch reconciliation and supervisor exit/removal events +stop a running workload whose companion is unavailable. The gateway also +requires the authenticated supervisor session before publishing Ready. + +## Mounts, GPUs, and configuration + +User `bind`, `volume`, `tmpfs`, and `image` mounts and CDI GPU selection remain +native Podman features and apply only to the workload. Bind mounts require the +operator's `enable_bind_mounts` opt-in. Reserved control paths and the workspace +root cannot be replaced. User-owned volumes are never created or deleted. + +See [gateway configuration](../../docs/reference/gateway-config.mdx) for +operator settings and [NETWORKING.md](NETWORKING.md) for callback networking. +The configured network and upstream proxy belong to the supervisor. +`health_check_interval_secs=0` uses a one-second check rather than disabling +the readiness check required by this topology. + +Gateway OTLP configuration continues to export compute-driver spans under the +`openshell-driver-podman` service, preserving gateway trace context. diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 0670a026dd..7ab13d0a42 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -177,6 +177,8 @@ pub struct ImageInspect { pub struct ImageConfig { #[serde(default)] pub user: String, + #[serde(default)] + pub env: Vec, } /// A container summary returned by the list API. @@ -450,6 +452,89 @@ impl PodmanClient { .await } + pub(crate) async fn create_typed_container( + &self, + spec: &(impl serde::Serialize + Sync), + ) -> Result { + #[derive(serde::Deserialize)] + struct Created { + #[serde(rename = "Id", alias = "ID")] + id: String, + } + let body = + serde_json::to_vec(spec).map_err(|error| PodmanApiError::Json(error.to_string()))?; + let (status, bytes) = self + .request_raw( + hyper::Method::POST, + "/libpod/containers/create", + "application/json", + body.into(), + ) + .await?; + if !status.is_success() { + return Err(error_from_response(status.as_u16(), &bytes)); + } + let created: Created = serde_json::from_slice(&bytes) + .map_err(|error| PodmanApiError::Json(error.to_string()))?; + validate_name(&created.id)?; + Ok(created.id) + } + + pub(crate) async fn copy_to_container( + &self, + name: &str, + archive: Vec, + ) -> Result<(), PodmanApiError> { + validate_name(name)?; + let (status, bytes) = self + .request_raw( + hyper::Method::PUT, + &format!("/libpod/containers/{name}/archive?path=/"), + "application/x-tar", + archive.into(), + ) + .await?; + if status.is_success() { + Ok(()) + } else { + Err(error_from_response(status.as_u16(), &bytes)) + } + } + + pub(crate) async fn verify_isolation_fence(&self, id: &str) -> Result<(), PodmanApiError> { + #[derive(serde::Deserialize)] + #[serde(rename_all = "PascalCase")] + struct HostConfig { + network_mode: String, + privileged: bool, + } + #[derive(serde::Deserialize)] + #[serde(rename_all = "PascalCase")] + struct FenceInspect { + host_config: HostConfig, + network_settings: NetworkSettings, + } + validate_name(id)?; + let inspected: FenceInspect = self + .request_json( + hyper::Method::GET, + &format!("/libpod/containers/{id}/json"), + None, + ) + .await?; + if inspected.host_config.network_mode != "none" + || inspected.host_config.privileged + || inspected + .network_settings + .networks + .keys() + .any(|name| name != "none") + { + return Err(PodmanApiError::InvalidInput("sandbox requires an unprivileged container with network mode none and no attached networks".into())); + } + Ok(()) + } + /// Start a container by name or ID. pub async fn start_container(&self, name: &str) -> Result<(), PodmanApiError> { validate_name(name)?; diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 541cb2f756..7091fbb7a2 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -197,7 +197,7 @@ pub fn short_id(id: &str) -> String { // --------------------------------------------------------------------------- #[derive(Serialize)] -struct ContainerSpec { +pub struct ContainerSpec { name: String, image: String, labels: BTreeMap, @@ -212,18 +212,19 @@ struct ContainerSpec { entrypoint: Vec, command: Vec, user: String, + #[serde(skip_serializing_if = "Vec::is_empty")] + groups: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + unsetenv: Vec, cap_drop: Vec, cap_add: Vec, no_new_privileges: bool, + #[serde(skip_serializing_if = "String::is_empty")] seccomp_profile_path: String, - /// Podman's container create API accepts `AppArmor` through the dedicated - /// `apparmor_profile` `SpecGenerator` field. This is not Docker's - /// `security_opt` representation. - #[serde(skip_serializing_if = "Option::is_none")] - apparmor_profile: Option, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + sysctl: BTreeMap, image_pull_policy: String, - #[serde(skip_serializing_if = "Option::is_none")] - healthconfig: Option, + healthconfig: HealthConfig, resource_limits: ResourceLimits, /// Env-type secrets: map of `ENV_VAR_NAME → secret_name`. /// Podman's libpod `SpecGenerator` uses `secret_env` (a flat map) for @@ -340,15 +341,8 @@ struct SecretMount { struct ResourceLimits { cpu: CpuLimits, memory: MemoryLimits, - // Podman's libpod API consumes the OCI LinuxResources shape. A Docker-style - // scalar PidsLimit is silently ignored and leaves the runtime default. - #[serde(skip_serializing_if = "Option::is_none")] - pids: Option, -} - -#[derive(Serialize)] -struct PidsLimits { - limit: i64, + #[serde(rename = "PidsLimit", skip_serializing_if = "Option::is_none")] + pids_limit: Option, } #[derive(Serialize)] @@ -486,7 +480,7 @@ fn build_env( config: &PodmanComputeConfig, image: &str, oci_user: &str, -) -> BTreeMap { +) -> Result, ComputeDriverError> { let spec = sandbox.spec.as_ref(); let template = spec.and_then(|s| s.template.as_ref()); @@ -511,10 +505,11 @@ fn build_env( user_env.insert(k.clone(), v.clone()); } } - env.extend(user_env.clone()); - if !user_env.is_empty() - && let Ok(json) = serde_json::to_string(&user_env) - { + // User environment belongs exclusively to mediated workload children. In + // particular, never activate loader or policy overrides in the supervisor. + if !user_env.is_empty() { + let json = serde_json::to_string(&user_env) + .map_err(|error| ComputeDriverError::Precondition(error.to_string()))?; env.insert(openshell_core::sandbox_env::USER_ENVIRONMENT.into(), json); } @@ -539,11 +534,11 @@ fn build_env( ); env.insert( openshell_core::sandbox_env::SSH_SOCKET_PATH.into(), - config.ssh_socket_path.clone(), + config.sandbox_ssh_socket_path.clone(), ); env.insert("OPENSHELL_CONTAINER_IMAGE".into(), image.to_string()); let main_process = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(spec) - .expect("main process config serialization cannot fail"); + .map_err(|error| ComputeDriverError::Precondition(error.to_string()))?; env.insert( openshell_core::sandbox_env::MAIN_PROCESS_SPEC.into(), main_process, @@ -615,7 +610,7 @@ fn build_env( ); } - env + Ok(env) } /// Merge labels from the sandbox template with required managed labels. @@ -665,12 +660,14 @@ fn build_resource_limits(sandbox: &DriverSandbox, config: &PodmanComputeConfig) period: DEFAULT_CPU_PERIOD, }, memory: MemoryLimits { limit: mem_bytes }, - pids: config - .sandbox_pids_limit - .map(|limit| PidsLimits { limit: limit.get() }), + pids_limit: podman_pids_limit(config.sandbox_pids_limit), } } +fn podman_pids_limit(value: i64) -> Option { + if value > 0 { Some(value) } else { None } +} + pub fn podman_driver_volume_mount_sources( sandbox: &DriverSandbox, enable_bind_mounts: bool, @@ -880,6 +877,7 @@ fn validate_podman_driver_mounts( } }; driver_mounts::validate_container_mount_target(target)?; + driver_mounts::validate_mount_control_path(target, "/.openshell")?; let normalized_target = driver_mounts::normalize_mount_target(target); if !targets.insert(normalized_target.clone()) { return Err(format!( @@ -951,14 +949,6 @@ fn validate_tmpfs_options(options: &[String]) -> Result, String> { .collect() } -fn podman_apparmor_profile(profile: Option<&openshell_core::AppArmorProfile>) -> Option { - match profile { - None | Some(openshell_core::AppArmorProfile::RuntimeDefault) => None, - Some(openshell_core::AppArmorProfile::Unconfined) => Some("unconfined".to_string()), - Some(openshell_core::AppArmorProfile::Localhost(profile)) => Some(profile.clone()), - } -} - /// Build the Podman container creation JSON spec. #[cfg(test)] #[must_use] @@ -1025,6 +1015,7 @@ pub fn build_container_spec_with_token_and_gpu_devices( } #[allow(clippy::too_many_arguments)] +#[cfg(test)] pub fn build_container_spec_for_image( sandbox: &DriverSandbox, config: &PodmanComputeConfig, @@ -1036,10 +1027,36 @@ pub fn build_container_spec_for_image( supervisor_bin_path: Option<&Path>, tls_secret_names: Option<&[String; 3]>, ) -> Result { + serde_json::to_value(build_base_spec( + sandbox, + config, + token_secret_name, + gpu_device_ids, + requested_image, + image_id, + oci_user, + supervisor_bin_path, + tls_secret_names, + )?) + .map_err(|error| ComputeDriverError::Message(format!("encode Podman spec: {error}"))) +} + +#[allow(clippy::too_many_arguments)] +fn build_base_spec( + sandbox: &DriverSandbox, + config: &PodmanComputeConfig, + token_secret_name: Option<&str>, + gpu_device_ids: Option<&[String]>, + requested_image: &str, + image_id: &str, + oci_user: &str, + supervisor_bin_path: Option<&Path>, + tls_secret_names: Option<&[String; 3]>, +) -> Result { let name = container_name(&sandbox.workspace, &sandbox.name, &sandbox.id); let vol = volume_name(&sandbox.id); - let env = build_env(sandbox, config, requested_image, oci_user); + let env = build_env(sandbox, config, requested_image, oci_user)?; let labels = build_labels(sandbox); let resource_limits = build_resource_limits(sandbox, config); let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts) @@ -1117,99 +1134,31 @@ pub fn build_container_spec_for_image( // corporate proxy flags follow it; the workload command comes from // the reserved environment variable. command, - // Force the supervisor to run as root (UID 0). Sandbox images may - // set a non-root USER directive (e.g. `USER sandbox`), but the - // supervisor needs root to create network namespaces, set up the - // proxy, and configure Landlock/seccomp. This matches the K8s - // driver's runAsUser: 0. - user: "0:0".into(), - // Podman's default container capability set is already restricted: - // CHOWN DAC_OVERRIDE FOWNER FSETID KILL SETGID SETUID SETPCAP - // NET_BIND_SERVICE SYS_CHROOT SETFCAP - // We add what the supervisor needs and drop what it doesn't. - cap_drop: vec![ - // Not needed: standard file permission bits are sufficient; dropping - // prevents the supervisor from bypassing DAC checks it shouldn't need. - "DAC_OVERRIDE".into(), - // Not needed: the supervisor does not create setuid/setgid executables. - "FSETID".into(), - // Not needed: the supervisor does not bind privileged ports (<1024). - "NET_BIND_SERVICE".into(), - // Not in Podman's default set but explicitly denied in case the image - // or runtime adds it; raw sockets are not required. - "NET_RAW".into(), - // Not needed: the supervisor does not manipulate file capabilities. - "SETFCAP".into(), - // Not needed: the supervisor does not call chroot(). - "SYS_CHROOT".into(), - ], - cap_add: vec![ - // seccomp filter installation, namespace creation, Landlock setup. - "SYS_ADMIN".into(), - // Network namespace veth setup, IP/route configuration. - "NET_ADMIN".into(), - // Reading /proc//exe and ancestor walk for process identity in policy. - "SYS_PTRACE".into(), - // Reading /dev/kmsg for bypass-detection diagnostics. - "SYSLOG".into(), - // Reading /proc//fd/ across UIDs for process identity resolution. - // In rootless Podman the supervisor runs as UID 0 inside a user namespace - // while sandbox processes run as the sandbox user. The kernel's - // proc_fd_permission() calls generic_permission() which denies cross-UID - // access to the dr-x------ fd directory unless this cap is present. - // Without it the proxy cannot determine which binary made each outbound - // connection and all traffic is denied. - "DAC_READ_SEARCH".into(), - // Child setup clears the capability bounding set before exec, which - // requires CAP_SETPCAP in the supervisor until drop_privileges(). - "SETPCAP".into(), - // Forwarding shutdown signals to the canonical workload process - // group after it drops to the sandbox UID requires CAP_KILL. - "KILL".into(), - ], - // SETUID, SETGID, SETPCAP, CHOWN, and FOWNER are intentionally kept from - // Podman's default set and not dropped: - // SETUID/SETGID – drop_privileges(): setuid()/setgid()/initgroups() to the - // sandbox user. In rootless Podman cap_drop:ALL removes them - // from the bounding set even though uid=0 owns the user - // namespace — so we keep them by not dropping them explicitly. - // SETPCAP – drop_privileges(): clears the child capability - // bounding set before the sandbox user execs. - // CHOWN – prepare_filesystem(): chown(path, uid, gid) on newly - // created read_write directories so the sandbox user can - // write to them. - // FOWNER – chown on files where the supervisor is not the owner - // (e.g. pre-existing directories owned by another user). - // - // Disable the container-level seccomp profile. The sandbox supervisor The sandbox supervisor - // installs its own policy-aware BPF seccomp filter at runtime via - // seccompiler (two-phase: clone3 blocker + main filter). The runtime - // filter is more restrictive than Podman's default — it blocks 20+ - // dangerous syscalls and conditionally restricts socket domains based - // on network policy. The filter self-seals by blocking further - // seccomp(SET_MODE_FILTER) calls after installation. - // - // A container-level profile would interfere by blocking the landlock - // and seccomp syscalls the supervisor needs during setup, before it - // locks itself down. + // The paired builder supplies the immutable non-root identity. + user: String::new(), + groups: Vec::new(), + unsetenv: Vec::new(), + cap_drop: vec!["ALL".into()], + cap_add: Vec::new(), no_new_privileges: true, - seccomp_profile_path: "unconfined".into(), - apparmor_profile: podman_apparmor_profile(config.app_armor_profile.as_ref()), + // Omission selects the runtime default, never an unconfined profile. + seccomp_profile_path: String::new(), + sysctl: BTreeMap::new(), image_pull_policy: "never".to_string(), - healthconfig: config.health_check_interval_secs.map(|interval_secs| HealthConfig { + healthconfig: HealthConfig { test: vec![ "CMD-SHELL".into(), format!( "test -e /var/run/openshell-ssh-ready || test -S {} || ss -tlnp | grep -q :{}", - config.ssh_socket_path, + config.sandbox_ssh_socket_path, openshell_core::config::DEFAULT_SSH_PORT ), ], - interval: interval_secs.get() * 1_000_000_000, + interval: config.health_check_interval_secs * 1_000_000_000, timeout: 2_000_000_000, retries: 10, start_period: 5_000_000_000, - }), + }, resource_limits, secret_env: BTreeMap::new(), secrets: { @@ -1409,7 +1358,185 @@ pub fn build_container_spec_for_image( }, }; - Ok(serde_json::to_value(container_spec).expect("ContainerSpec serialization cannot fail")) + Ok(container_spec) +} + +/// Driver-owned inputs for the two independent runtime containers. +pub struct IsolationSpecInput<'a> { + pub sandbox: &'a DriverSandbox, + pub config: &'a PodmanComputeConfig, + pub token_secret: Option<&'a str>, + pub gpu_devices: Option<&'a [String]>, + pub requested_image: &'a str, + pub image_id: &'a str, + pub image_user: &'a str, + pub image_env: &'a [String], + pub supervisor_bin: Option<&'a Path>, + pub tls_secrets: Option<&'a [String; 3]>, + pub identity: &'a openshell_isolation_interface::contract::ResolvedWorkloadIdentity, +} + +pub struct IsolationSpecs { + pub workload: ContainerSpec, + pub supervisor: ContainerSpec, +} + +impl ContainerSpec { + pub(crate) fn join_user_namespace(&mut self, container_id: &str) { + self.userns = Some(UserNS { + nsmode: "container".to_string(), + value: Some(container_id.to_string()), + }); + self.idmappings = None; + } +} + +pub fn build_isolation_specs( + input: IsolationSpecInput<'_>, +) -> Result { + let base = || { + build_base_spec( + input.sandbox, + input.config, + input.token_secret, + input.gpu_devices, + input.requested_image, + input.image_id, + input.image_user, + input.supervisor_bin, + input.tls_secrets, + ) + }; + let mut workload = base()?; + let mut supervisor = base()?; + let user = format!("{}:{}", input.identity.uid, input.identity.gid); + let channel = crate::isolation::channel_volume_name(&input.sandbox.id); + + workload + .labels + .insert(crate::isolation::LABEL_ROLE.into(), "sandbox".into()); + workload.env = BTreeMap::new(); + workload.unsetenv = input + .image_env + .iter() + .filter_map(|entry| entry.split_once('=').map(|(key, _)| key.to_string())) + .collect(); + workload.command = vec![ + "--bootstrap".into(), + crate::isolation::BOOTSTRAP_PATH.into(), + ]; + workload.user.clone_from(&user); + workload.groups = input + .identity + .supplementary_gids + .iter() + .map(ToString::to_string) + .collect(); + workload.cap_drop = vec!["ALL".into()]; + workload.cap_add.clear(); + workload.seccomp_profile_path.clear(); + workload + .sysctl + .insert("net.ipv4.ip_unprivileged_port_start".into(), "0".into()); + workload.netns.nsmode = "none".into(); + workload.networks.clear(); + workload.portmappings.clear(); + workload.hostadd.clear(); + workload.secret_env.clear(); + workload.secrets.clear(); + workload.healthconfig.test = vec!["NONE".into()]; + workload + .mounts + .retain(|mount| !trusted_mount(&mount.destination)); + workload.volumes.push(NamedVolume { + name: channel.clone(), + dest: crate::isolation::CHANNEL_ROOT.into(), + options: vec!["rw".into(), "nocopy".into(), "z".into()], + }); + + supervisor.name = crate::isolation::supervisor_name(&input.sandbox.id); + supervisor + .labels + .insert(crate::isolation::LABEL_ROLE.into(), "supervisor".into()); + supervisor.image.clone_from(&input.config.supervisor_image); + supervisor.entrypoint = vec!["/openshell-supervisor".into()]; + supervisor.command.extend([ + "--topology-backend-name=podman".into(), + format!( + "--topology-payload-file={}", + crate::isolation::TOPOLOGY_PATH + ), + "--health-socket-path=/run/openshell/supervisor-health.sock".into(), + ]); + supervisor.env.insert( + openshell_core::sandbox_env::ADMITTED_ISOLATION_BACKEND.into(), + "podman".into(), + ); + supervisor.user = user; + supervisor.groups = input + .identity + .supplementary_gids + .iter() + .map(ToString::to_string) + .collect(); + supervisor.cap_drop = vec!["ALL".into()]; + supervisor.cap_add.clear(); + supervisor.seccomp_profile_path.clear(); + supervisor.devices = None; + supervisor.image_volumes.clear(); + supervisor.volumes = vec![NamedVolume { + name: channel, + dest: crate::isolation::CHANNEL_ROOT.into(), + options: vec!["ro".into(), "nocopy".into(), "z".into()], + }]; + supervisor.mounts.retain(|mount| { + trusted_mount(&mount.destination) + && mount.destination != openshell_core::container_paths::NETNS_MOUNT_ROOT + }); + for destination in ["/run", "/var/log", "/tmp"] { + supervisor.mounts.push(Mount { + kind: "tmpfs".into(), + source: "tmpfs".into(), + destination: destination.into(), + options: vec![ + "rw".into(), + "nosuid".into(), + "nodev".into(), + format!("uid={}", input.identity.uid), + format!("gid={}", input.identity.gid), + "mode=0700".into(), + "size=64m".into(), + ], + }); + } + for secret in &mut supervisor.secrets { + secret.uid = input.identity.uid; + secret.gid = input.identity.gid; + } + supervisor.healthconfig.test = vec![ + "CMD".into(), + "/openshell-supervisor".into(), + "health".into(), + "--socket".into(), + "/run/openshell/supervisor-health.sock".into(), + ]; + supervisor.healthconfig.interval = + input.config.health_check_interval_secs.max(1) * 1_000_000_000; + Ok(IsolationSpecs { + workload, + supervisor, + }) +} + +fn trusted_mount(destination: &str) -> bool { + matches!( + destination, + TLS_CA_MOUNT_PATH + | TLS_CERT_MOUNT_PATH + | TLS_KEY_MOUNT_PATH + | PROXY_CA_MOUNT_PATH + | PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR + ) || destination == openshell_core::container_paths::NETNS_MOUNT_ROOT } fn provider_spiffe_workload_api_socket_env_value(config: &PodmanComputeConfig) -> Option { @@ -1516,6 +1643,70 @@ mod tests { static ENV_LOCK: std::sync::LazyLock> = std::sync::LazyLock::new(|| std::sync::Mutex::new(())); + #[test] + fn isolated_pair_keeps_privileges_network_and_secrets_out_of_workload() { + let sandbox = DriverSandbox { + id: "pair".into(), + name: "agent".into(), + ..Default::default() + }; + let config = PodmanComputeConfig::default(); + let identity = openshell_isolation_interface::contract::ResolvedWorkloadIdentity::new( + 1000, + 1001, + vec![2000], + "image".into(), + "sha256:image".into(), + ) + .unwrap(); + let env = vec![ + "LD_PRELOAD=/hostile.so".into(), + "HTTP_PROXY=http://bypass".into(), + ]; + let specs = build_isolation_specs(IsolationSpecInput { + sandbox: &sandbox, + config: &config, + token_secret: Some("jwt"), + gpu_devices: None, + requested_image: "image:latest", + image_id: "sha256:image", + image_user: "1000:1001", + image_env: &env, + supervisor_bin: None, + tls_secrets: None, + identity: &identity, + }) + .unwrap(); + for spec in [&specs.workload, &specs.supervisor] { + assert_eq!(spec.user, "1000:1001"); + assert_eq!(spec.groups, vec!["2000"]); + assert_eq!(spec.cap_drop, vec!["ALL"]); + assert!(spec.cap_add.is_empty()); + assert!(spec.seccomp_profile_path.is_empty()); + assert!(spec.no_new_privileges); + } + assert_eq!(specs.workload.netns.nsmode, "none"); + assert!(specs.workload.networks.is_empty()); + assert!(specs.workload.portmappings.is_empty()); + assert!(specs.workload.env.is_empty()); + assert_eq!(specs.workload.unsetenv, vec!["LD_PRELOAD", "HTTP_PROXY"]); + assert!(specs.workload.secrets.is_empty()); + assert!( + specs + .workload + .mounts + .iter() + .all(|mount| !trusted_mount(&mount.destination)) + ); + assert_eq!(specs.supervisor.secrets.len(), 1); + assert_eq!(specs.supervisor.secrets[0].source, "jwt"); + assert_eq!(specs.supervisor.secrets[0].uid, 1000); + assert_eq!(specs.supervisor.volumes.len(), 1); + assert!(specs.supervisor.volumes[0].options.contains(&"ro".into())); + assert!(specs.supervisor.volumes[0].options.contains(&"z".into())); + assert_eq!(specs.supervisor.entrypoint, vec!["/openshell-supervisor"]); + } + fn json_struct(value: Value) -> prost_types::Struct { let Value::Object(object) = value else { panic!("expected JSON object"); @@ -1564,7 +1755,7 @@ mod tests { } #[test] - fn container_spec_applies_resource_limits() { + fn container_spec_applies_cpu_and_memory_limits() { use openshell_core::proto::compute::v1::{ DriverResourceRequirements, DriverSandboxSpec, DriverSandboxTemplate, }; @@ -1581,8 +1772,7 @@ mod tests { }), ..Default::default() }); - let mut config = test_config(); - config.sandbox_pids_limit = std::num::NonZeroI64::new(2048); + let config = test_config(); let spec = build_container_spec(&sandbox, &config); assert_eq!( @@ -1594,8 +1784,8 @@ mod tests { Some(2 * 1024 * 1024 * 1024) ); assert_eq!( - spec["resource_limits"]["pids"]["limit"].as_i64(), - Some(2048) + spec["resource_limits"]["PidsLimit"].as_i64(), + Some(crate::config::DEFAULT_SANDBOX_PIDS_LIMIT) ); } @@ -1603,44 +1793,10 @@ mod tests { fn container_spec_can_inherit_runtime_pids_limit() { let sandbox = test_sandbox("test-id", "test-name"); let mut config = test_config(); - config.sandbox_pids_limit = None; + config.sandbox_pids_limit = 0; let spec = build_container_spec(&sandbox, &config); - assert!(spec["resource_limits"].get("pids").is_none()); - } - - #[test] - fn container_spec_uses_podman_apparmor_profile_field() { - let sandbox = test_sandbox("test-id", "test-name"); - - for (profile, expected) in [ - (openshell_core::AppArmorProfile::Unconfined, "unconfined"), - ( - openshell_core::AppArmorProfile::Localhost("openshell-supervisor".to_string()), - "openshell-supervisor", - ), - ] { - let mut config = test_config(); - config.app_armor_profile = Some(profile); - let spec = build_container_spec(&sandbox, &config); - - assert_eq!(spec["apparmor_profile"].as_str(), Some(expected)); - assert!(spec.get("security_opt").is_none()); - } - } - - #[test] - fn container_spec_omits_podman_apparmor_profile_for_runtime_default() { - let sandbox = test_sandbox("test-id", "test-name"); - - for profile in [None, Some(openshell_core::AppArmorProfile::RuntimeDefault)] { - let mut config = test_config(); - config.app_armor_profile = profile; - let spec = build_container_spec(&sandbox, &config); - - assert!(spec.get("apparmor_profile").is_none()); - assert!(spec.get("security_opt").is_none()); - } + assert!(spec["resource_limits"].get("PidsLimit").is_none()); } #[test] @@ -1681,7 +1837,8 @@ mod tests { container["env"]["OPENSHELL_CONTAINER_IMAGE"].as_str(), Some("registry.example/app:latest") ); - assert_eq!(container["user"].as_str(), Some("0:0")); + // Only the paired builder materializes the immutable non-root user. + assert_eq!(container["user"].as_str(), Some("")); assert_eq!(container["image_pull_policy"].as_str(), Some("never")); assert_eq!(container["dns_search"], serde_json::json!([])); assert_eq!(container["dns_option"], serde_json::json!([])); @@ -1921,66 +2078,25 @@ mod tests { } #[test] - fn container_spec_includes_required_capabilities() { + fn container_spec_defaults_drop_capabilities_and_keep_runtime_seccomp() { let sandbox = test_sandbox("test-id", "test-name"); let config = test_config(); - let spec = build_container_spec(&sandbox, &config); - - let added: Vec<&str> = spec["cap_add"] - .as_array() - .expect("cap_add should be an array") - .iter() - .filter_map(|v| v.as_str()) - .collect(); - assert!(added.contains(&"SYS_ADMIN"), "missing SYS_ADMIN"); - assert!(added.contains(&"NET_ADMIN"), "missing NET_ADMIN"); - assert!(added.contains(&"SYS_PTRACE"), "missing SYS_PTRACE"); - assert!(added.contains(&"SYSLOG"), "missing SYSLOG"); - assert!( - added.contains(&"DAC_READ_SEARCH"), - "missing DAC_READ_SEARCH" - ); - assert!(added.contains(&"SETPCAP"), "missing SETPCAP"); - assert!(added.contains(&"KILL"), "missing KILL"); - - // SETUID and SETGID are NOT in cap_add — they remain available from the - // default bounding set because we no longer use cap_drop:ALL. Verify they - // are also not explicitly dropped. Similarly SETPCAP, CHOWN and FOWNER - // must not be dropped because child setup clears the bounding set and - // prepare_filesystem() calls chown() on newly created read_write - // directories before the supervisor drops privileges. - let dropped: Vec<&str> = spec["cap_drop"] - .as_array() - .expect("cap_drop should be an array") - .iter() - .filter_map(|v| v.as_str()) - .collect(); - assert!(!dropped.contains(&"SETUID"), "SETUID must not be dropped"); - assert!(!dropped.contains(&"SETGID"), "SETGID must not be dropped"); - assert!( - dropped.contains(&"NET_BIND_SERVICE"), - "NET_BIND_SERVICE must stay dropped; policy DNS binds an unprivileged port" - ); - assert!( - !dropped.contains(&"CHOWN"), - "CHOWN must not be dropped (needed for prepare_filesystem chown)" - ); - assert!( - !dropped.contains(&"FOWNER"), - "FOWNER must not be dropped (needed for chown on non-owned files)" - ); - assert!( - !dropped.contains(&"SETPCAP"), - "SETPCAP must not be dropped (needed for child bounding-set clear)" - ); - assert!( - !dropped.contains(&"KILL"), - "KILL must not be dropped (needed to signal the sandbox workload on shutdown)" - ); - assert!( - !dropped.contains(&"ALL"), - "must not use cap_drop:ALL in rootless Podman" - ); + let spec = build_base_spec( + &sandbox, + &config, + None, + None, + "image", + "sha256:image", + "", + None, + None, + ) + .unwrap(); + assert_eq!(spec.cap_drop, vec!["ALL"]); + assert!(spec.cap_add.is_empty()); + assert!(spec.seccomp_profile_path.is_empty()); + assert!(spec.no_new_privileges); } #[test] @@ -2016,8 +2132,7 @@ mod tests { #[test] fn container_spec_healthcheck_accepts_supervisor_socket() { let sandbox = test_sandbox("test-id", "test-name"); - let mut config = test_config(); - config.health_check_interval_secs = std::num::NonZeroU64::new(10); + let config = test_config(); let spec = build_container_spec(&sandbox, &config); let healthcheck = spec["healthconfig"]["test"] @@ -2033,18 +2148,11 @@ mod tests { ); } - #[test] - fn container_spec_omits_healthcheck_when_disabled() { - let sandbox = test_sandbox("test-id", "test-name"); - let spec = build_container_spec(&sandbox, &test_config()); - assert!(spec.get("healthconfig").is_none()); - } - #[test] fn container_spec_healthcheck_interval_from_config() { let sandbox = test_sandbox("test-id", "test-name"); let mut config = test_config(); - config.health_check_interval_secs = std::num::NonZeroU64::new(30); + config.health_check_interval_secs = 30; let spec = build_container_spec(&sandbox, &config); let interval = spec["healthconfig"]["Interval"] @@ -2470,7 +2578,7 @@ mod tests { default_image: "test-image:latest".to_string(), grpc_endpoint: "http://localhost:50051".to_string(), host_gateway_ip: String::new(), - ssh_socket_path: "/run/openshell/test-ssh.sock".to_string(), + sandbox_ssh_socket_path: "/run/openshell/test-ssh.sock".to_string(), ..PodmanComputeConfig::default() } } @@ -2930,6 +3038,27 @@ mod tests { assert!(err.to_string().contains("reserved OpenShell path")); } + #[test] + fn user_mounts_cannot_replace_private_channel_hierarchy() { + for target in [ + "/.openshell", + "/.openshell/channel", + "/.openshell/channel/sandbox", + "/.openshell/supervisor", + ] { + let mount = PodmanDriverMountConfig::Volume { + source: "user-owned".into(), + target: target.into(), + read_only: false, + subpath: None, + }; + assert!( + validate_podman_driver_mounts(&[mount], false).is_err(), + "{target}" + ); + } + } + #[test] fn container_spec_uses_configured_host_gateway_ip() { let sandbox = test_sandbox("test-id", "test-name"); diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index cea9268fef..f7d20cb374 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -777,7 +777,7 @@ impl PodmanComputeDriver { "Creating sandbox container" ); - let (image, immutable_image_id, image_user) = async { + let (image, immutable_image_id, image_user, image_env) = async { let phase_status = openshell_otel::ErrorStatusGuard::current(); let result = async { // The supervisor binary is shipped in a standalone OCI image and @@ -837,7 +837,8 @@ impl PodmanComputeDriver { .map_err(ComputeDriverError::from)?; } - Ok((image.to_string(), inspected_image.id, image_user)) + let image_env = inspected_image.config.as_ref().map_or_else(Vec::new, |config| config.env.clone()); + Ok((image.to_string(), inspected_image.id, image_user, image_env)) } .await; phase_status.finish(result) @@ -856,6 +857,22 @@ impl PodmanComputeDriver { // content at startup. validate_sandbox_proxy_ca_bundle(&self.config).await?; + let identity = self + .resolve_workload_identity(sandbox, &immutable_image_id, &image_user) + .await?; + let channel_volume = crate::isolation::channel_volume_name(&sandbox.id); + let mut runtime_config = self.config.clone(); + runtime_config.supervisor_image = self + .client + .inspect_image(&self.config.supervisor_image) + .await? + .id; + if runtime_config.supervisor_image.is_empty() { + return Err(ComputeDriverError::Precondition( + "supervisor image inspection returned no immutable image ID".into(), + )); + } + // Create workspace volume and per-sandbox token secret. let (token_secret_name, proxy_auth_secret_name) = async { let phase_status = openshell_otel::ErrorStatusGuard::current(); @@ -901,6 +918,7 @@ impl PodmanComputeDriver { // Clean up the volume and both per-sandbox secrets on any failure past // this point. let cleanup_created = || async { + let _ = self.client.remove_volume(&channel_volume).await; let _ = self.client.remove_volume(&vol_name).await; if let Some(secret) = token_secret_name.as_deref() { cleanup_sandbox_token_secret(&self.client, secret).await; @@ -911,7 +929,7 @@ impl PodmanComputeDriver { }; // Prepare and create the container. - let tls_secret_names = async { + async { let phase_status = openshell_otel::ErrorStatusGuard::current(); let result = async { let gpu_devices = match self.resolve_gpu_cdi_devices( @@ -927,7 +945,7 @@ impl PodmanComputeDriver { }; let supervisor_bin_path = if userns_needs_extraction(self.config.userns.as_deref()) { - match extract_supervisor_bin(&self.client, &self.config).await { + match extract_supervisor_bin(&self.client, &runtime_config).await { Ok(path) => Some(path), Err(e) => { cleanup_created().await; @@ -938,9 +956,7 @@ impl PodmanComputeDriver { None }; - let tls_secret_names = if userns_remaps_uids(self.config.userns.as_deref()) - && self.config.tls_enabled() - { + let tls_secret_names = if self.config.tls_enabled() { let names = container::tls_secret_names(&sandbox.id); if let Err(e) = create_tls_secrets(&self.client, &self.config, &names).await { cleanup_created().await; @@ -958,32 +974,77 @@ impl PodmanComputeDriver { } }; - let spec = match container::build_container_spec_for_image( + let specs = container::build_isolation_specs(container::IsolationSpecInput { sandbox, - &self.config, - token_secret_name.as_deref(), - gpu_devices.as_deref(), - &image, - &immutable_image_id, - &image_user, - supervisor_bin_path.as_deref(), - tls_secret_names.as_ref(), - ) { + config: &runtime_config, + token_secret: token_secret_name.as_deref(), + gpu_devices: gpu_devices.as_deref(), + requested_image: &image, + image_id: &immutable_image_id, + image_user: &image_user, + image_env: &image_env, + supervisor_bin: supervisor_bin_path.as_deref(), + tls_secrets: tls_secret_names.as_ref(), + identity: &identity, + }); + let mut specs = match specs { Ok(spec) => spec, Err(e) => { cleanup_all().await; return Err(e); } }; - match self.client.create_container(&spec).await { - Ok(_) => Ok(tls_secret_names), - Err(PodmanApiError::Conflict(_)) => { - cleanup_all().await; - Err(ComputeDriverError::AlreadyExists) + let mut created_workload = None; + let mut created_supervisor = None; + let create_result = async { + self.client.create_volume(&channel_volume).await?; + let workload_id = self.client.create_typed_container(&specs.workload).await?; + created_workload = Some(workload_id.clone()); + self.client.verify_isolation_fence(&workload_id).await?; + let child_env = image_env + .iter() + .filter_map(|entry| { + entry + .split_once('=') + .map(|(key, value)| (key.into(), value.into())) + }) + .collect(); + let archives = crate::isolation::bootstrap_archives( + &sandbox.id, + &workload_id, + &identity, + child_env, + )?; + self.client + .copy_to_container(&workload_id, archives.workload) + .await?; + specs.supervisor.join_user_namespace(&workload_id); + let supervisor_id = self + .client + .create_typed_container(&specs.supervisor) + .await?; + created_supervisor = Some(supervisor_id.clone()); + self.client + .copy_to_container(&supervisor_id, archives.supervisor) + .await?; + // Both resources and private files exist before either + // container can run. Only the trusted sandbox starts here; + // authenticated confirmation gates subsequent agent exec. + self.client.start_container(&workload_id).await?; + self.client.start_container(&supervisor_id).await?; + Ok::<(), ComputeDriverError>(()) + } + .await; + if create_result.is_err() { + for id in [created_supervisor, created_workload].into_iter().flatten() { + let _ = self.client.remove_container(&id, 0).await; } + } + match create_result { + Ok(()) => Ok(()), Err(e) => { cleanup_all().await; - Err(ComputeDriverError::from(e)) + Err(e) } } } @@ -998,44 +1059,6 @@ impl PodmanComputeDriver { )) .await?; - let cleanup_all = || async { - cleanup_created().await; - if let Some(names) = &tls_secret_names { - cleanup_tls_secrets(&self.client, names).await; - } - }; - - // Start container. - let start_result = async { - let phase_status = openshell_otel::ErrorStatusGuard::current(); - let result = self - .client - .start_container(&name) - .await - .map_err(ComputeDriverError::from); - phase_status.finish(result) - } - .instrument(tracing::info_span!( - "podman.start_container", - otel.name = "podman.start_container", - otel.status_code = tracing::field::Empty, - container.name = %name, - )) - .await; - if let Err(e) = start_result { - warn!( - sandbox_name = %sandbox.name, - error = %e, - "Failed to start container; cleaning up" - ); - let _ = self - .client - .remove_container(&name, self.config.stop_timeout_secs) - .await; - cleanup_all().await; - return Err(e); - } - info!( sandbox_id = %sandbox.id, sandbox_name = %sandbox.name, @@ -1045,7 +1068,65 @@ impl PodmanComputeDriver { span_status.finish(Ok(())) } - /// Find the Podman container ID for a sandbox by its sandbox ID using label lookup. + /// Resolve image accounts without executing any image-supplied program. + async fn resolve_workload_identity( + &self, + sandbox: &DriverSandbox, + image: &str, + image_user: &str, + ) -> Result + { + #[derive(serde::Serialize)] + struct InspectionSpec<'a> { + name: String, + image: &'a str, + } + // Inspect a stopped, unexecuted container pinned to the final image ID. + let id = self + .client + .create_typed_container(&InspectionSpec { + name: format!("openshell-identity-{}", uuid::Uuid::new_v4()), + image, + }) + .await?; + let result = + async { + // Do not let image-controlled symlinks alias the protected channel + // into an agent-readable subtree before Podman mounts it. + match self.client.copy_from_container(&id, "/.openshell").await { + Err(PodmanApiError::NotFound(_)) => {} + Ok(_) => return Err(ComputeDriverError::Precondition( + "workload images must not prepopulate the reserved /.openshell hierarchy" + .into(), + )), + Err(error) => return Err(error.into()), + } + let mut accounts = Vec::new(); + for path in ["/etc/passwd", "/etc/group"] { + let content = match self.client.copy_from_container(&id, path).await { + Ok(archive) => extract_first_tar_entry(&archive) + .map_err(ComputeDriverError::Precondition)?, + Err(PodmanApiError::NotFound(_)) => Vec::new(), + Err(error) => return Err(error.into()), + }; + accounts.push(content); + } + let [passwd, group] = accounts.as_slice() else { + return Err(ComputeDriverError::Precondition( + "image account inspection was incomplete".into(), + )); + }; + crate::isolation::resolve_identity(sandbox, image, image_user, passwd, group) + } + .await; + let cleanup = self.client.remove_container(&id, 0).await; + if let Err(error) = cleanup { + warn!(container = %id, %error, "Failed to remove stopped identity inspection container"); + } + result + } + + /// Find only the workload, never its supervisor companion. async fn find_container_id( &self, sandbox_id: &str, @@ -1060,7 +1141,11 @@ impl PodmanComputeDriver { let id_filter = format!("{LABEL_SANDBOX_ID}={sandbox_id}"); let entries = self .client - .list_containers(&[LABEL_MANAGED_FILTER, &id_filter]) + .list_containers(&[ + LABEL_MANAGED_FILTER, + &id_filter, + crate::isolation::WORKLOAD_FILTER, + ]) .await .map_err(ComputeDriverError::from)?; Ok(entries.into_iter().next()) @@ -1113,6 +1198,15 @@ impl PodmanComputeDriver { .await? .ok_or(ComputeDriverError::NotFound)?; let container_id = container.id; + let supervisor = crate::isolation::supervisor_name(sandbox_id); + match self + .client + .stop_container(&supervisor, self.config.stop_timeout_secs) + .await + { + Ok(()) | Err(PodmanApiError::NotFound(_)) => {} + Err(error) => return Err(error.into()), + } if container.state == "stopping" { let result = async { let finished_at = self @@ -1176,7 +1270,19 @@ impl PodmanComputeDriver { .await? .ok_or(ComputeDriverError::NotFound)?; if container.state == "running" { - return span_status.finish(Ok(())); + let supervisor = self + .client + .inspect_container(&crate::isolation::supervisor_name(sandbox_id)) + .await; + if supervisor + .as_ref() + .is_ok_and(|inspect| inspect.state.running) + { + return span_status.finish(Ok(())); + } + self.client.stop_container(&container.id, 0).await?; + self.wait_for_container_stopped(sandbox_id, &container.id) + .await?; } let container_id = container.id; info!(sandbox_id = %sandbox_id, container = %container_id, "Starting sandbox container"); @@ -1193,11 +1299,29 @@ impl PodmanComputeDriver { .map_err(ComputeDriverError::from)?; self.lifecycle_event_fences .record_previous_exit(sandbox_id, previous.state.finished_at.as_deref()); - let result = self - .client - .start_container(&container_id) - .await - .map_err(ComputeDriverError::from); + let result = async { + let supervisor = crate::isolation::supervisor_name(sandbox_id); + self.client + .stop_container(&supervisor, self.config.stop_timeout_secs) + .await?; + self.wait_for_container_stopped(sandbox_id, &supervisor) + .await?; + let archive = self + .client + .copy_from_container(&supervisor, crate::isolation::RESTART_BUNDLE_PATH) + .await?; + let bundle = + extract_first_tar_entry(&archive).map_err(ComputeDriverError::Precondition)?; + self.client.copy_to_container(&container_id, bundle).await?; + self.client.verify_isolation_fence(&container_id).await?; + self.client.start_container(&container_id).await?; + if let Err(error) = self.client.start_container(&supervisor).await { + let _ = self.client.stop_container(&container_id, 0).await; + return Err(error.into()); + } + Ok(()) + } + .await; span_status.finish(result) } @@ -1219,6 +1343,25 @@ impl PodmanComputeDriver { )); } + let supervisor = crate::isolation::supervisor_name(sandbox_id); + match self + .client + .remove_container(&supervisor, self.config.stop_timeout_secs) + .await + { + Ok(()) | Err(PodmanApiError::NotFound(_)) => {} + Err(error) => return Err(error.into()), + } + match self + .client + .remove_volume(&crate::isolation::channel_volume_name(sandbox_id)) + .await + { + Ok(()) | Err(PodmanApiError::NotFound(_)) => {} + // The workload still owns the volume until its removal below. + Err(error) => debug!(%error, "Channel volume is still attached to workload"), + } + let Some(container_id) = self.find_container_id(sandbox_id).await? else { debug!(sandbox_id = %sandbox_id, "Sandbox container not found (already deleted)"); let vol = container::volume_name(sandbox_id); @@ -1252,6 +1395,13 @@ impl PodmanComputeDriver { }; // Remove workspace volume. + if let Err(error) = self + .client + .remove_volume(&crate::isolation::channel_volume_name(sandbox_id)) + .await + { + warn!(%sandbox_id, %error, "Failed to remove private channel volume"); + } let vol = container::volume_name(sandbox_id); if let Err(e) = self.client.remove_volume(&vol).await { warn!( @@ -1278,7 +1428,11 @@ impl PodmanComputeDriver { let id_filter = format!("{LABEL_SANDBOX_ID}={sandbox_id}"); let entries = self .client - .list_containers(&[LABEL_MANAGED_FILTER, &id_filter]) + .list_containers(&[ + LABEL_MANAGED_FILTER, + &id_filter, + crate::isolation::WORKLOAD_FILTER, + ]) .await .map_err(ComputeDriverError::from)?; Ok(!entries.is_empty()) @@ -1292,16 +1446,18 @@ impl PodmanComputeDriver { let id_filter = format!("{LABEL_SANDBOX_ID}={sandbox_id}"); let entries = self .client - .list_containers(&[LABEL_MANAGED_FILTER, &id_filter]) + .list_containers(&[ + LABEL_MANAGED_FILTER, + &id_filter, + crate::isolation::WORKLOAD_FILTER, + ]) .await .map_err(ComputeDriverError::from)?; let Some(entry) = entries.first() else { return Ok(None); }; if entry.state == "running" { - Ok(self - .client - .inspect_container(&entry.id) + Ok(watcher::inspect_workload(&self.client, &entry.id) .await .ok() .and_then(|inspect| driver_sandbox_from_inspect(&inspect)) @@ -1318,7 +1474,7 @@ impl PodmanComputeDriver { pub async fn list_sandboxes(&self) -> Result, ComputeDriverError> { let entries = self .client - .list_containers(&[LABEL_MANAGED_FILTER]) + .list_containers(&[LABEL_MANAGED_FILTER, crate::isolation::WORKLOAD_FILTER]) .await .map_err(ComputeDriverError::from)?; @@ -1326,7 +1482,7 @@ impl PodmanComputeDriver { for entry in &entries { if entry.state == "running" { // Running containers need inspect for health check status. - match self.client.inspect_container(&entry.id).await { + match watcher::inspect_workload(&self.client, &entry.id).await { Ok(inspect) => { if let Some(sandbox) = driver_sandbox_from_inspect(&inspect) { sandboxes.push(sandbox); @@ -1609,6 +1765,7 @@ fn userns_needs_extraction(userns: Option<&str>) -> bool { /// Returns `true` when userns remaps all UIDs, making host-owned bind mounts /// unreadable from inside the container. `auto` and `no-map` remap every UID; /// `keep-id` preserves the host user's UID; `host` uses the host namespace. +#[cfg(test)] fn userns_remaps_uids(userns: Option<&str>) -> bool { userns.is_some_and(|mode| { let base = mode.split(':').next().unwrap_or(mode); @@ -1705,6 +1862,7 @@ mod tests { "lifecycle-stop", vec![ StubResponse::new(StatusCode::OK, r#"[{"Id":"ctr-1","State":"running"}]"#), + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion stop StubResponse::new(StatusCode::NO_CONTENT, ""), StubResponse::new( StatusCode::OK, @@ -1720,7 +1878,7 @@ mod tests { assert_eq!( stop_requests .lock() - .expect("request log lock should not be poisoned")[1], + .expect("request log lock should not be poisoned")[2], format!( "POST {}", api_path("/libpod/containers/ctr-1/stop?timeout=10") @@ -1729,7 +1887,7 @@ mod tests { assert_eq!( stop_requests .lock() - .expect("request log lock should not be poisoned")[2], + .expect("request log lock should not be poisoned")[3], format!("GET {}", api_path("/libpod/containers/ctr-1/json")) ); @@ -1741,8 +1899,7 @@ mod tests { StatusCode::OK, r#"{"Id":"ctr-1","Name":"sandbox","State":{"Status":"exited","Running":false,"FinishedAt":"2026-08-12T16:39:13Z"},"Config":{}}"#, ), - StubResponse::new(StatusCode::NO_CONTENT, ""), - ], + ].into_iter().chain(restart_responses()).collect(), ); test_driver(start_socket.clone()) .start_sandbox("sandbox-1") @@ -1759,7 +1916,10 @@ mod tests { start_requests .lock() .expect("request log lock should not be poisoned")[2], - format!("POST {}", api_path("/libpod/containers/ctr-1/start")) + format!( + "POST {}", + api_path("/libpod/containers/openshell-supervisor-sandbox-1/stop?timeout=10") + ) ); let _ = fs::remove_file(stop_socket); @@ -1772,6 +1932,7 @@ mod tests { "lifecycle-stop-wait", vec![ StubResponse::new(StatusCode::OK, r#"[{"Id":"ctr-1","State":"running"}]"#), + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion stop StubResponse::new(StatusCode::NO_CONTENT, ""), StubResponse::new( StatusCode::OK, @@ -1793,12 +1954,12 @@ mod tests { let requests = requests .lock() .expect("request log lock should not be poisoned"); - assert_eq!(requests.len(), 4); + assert_eq!(requests.len(), 5); assert_eq!( - requests[2], + requests[3], format!("GET {}", api_path("/libpod/containers/ctr-1/json")) ); - assert_eq!(requests[3], requests[2]); + assert_eq!(requests[4], requests[3]); let _ = fs::remove_file(socket); } @@ -1809,6 +1970,7 @@ mod tests { "lifecycle-stop-retry", vec![ StubResponse::new(StatusCode::OK, r#"[{"Id":"ctr-1","State":"stopping"}]"#), + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion stop StubResponse::new( StatusCode::OK, r#"{"Id":"ctr-1","Name":"sandbox","State":{"Status":"exited","Running":false,"FinishedAt":"2026-08-12T16:39:13Z"},"Config":{}}"#, @@ -1825,9 +1987,9 @@ mod tests { let requests = requests .lock() .expect("request log lock should not be poisoned"); - assert_eq!(requests.len(), 2); + assert_eq!(requests.len(), 3); assert_eq!( - requests[1], + requests[2], format!("GET {}", api_path("/libpod/containers/ctr-1/json")) ); @@ -1845,6 +2007,7 @@ mod tests { "trace-stop", vec![ StubResponse::new(StatusCode::OK, r#"[{"Id":"ctr-1","State":"running"}]"#), + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion stop StubResponse::new(StatusCode::NO_CONTENT, ""), StubResponse::new( StatusCode::OK, @@ -1893,17 +2056,10 @@ mod tests { let _tracing_lock = openshell_otel_test_support::tracing_test_lock().await; let (socket_path, _requests, handle) = spawn_podman_stub( "trace-create", - vec![ - StubResponse::new(StatusCode::OK, "{}"), - StubResponse::new(StatusCode::OK, "{}"), - StubResponse::new( - StatusCode::OK, - r#"{"Id":"sha256:sandbox","Config":{"User":"1234:1235"}}"#, - ), - StubResponse::new(StatusCode::CREATED, "{}"), - StubResponse::new(StatusCode::CREATED, "{}"), - StubResponse::new(StatusCode::NO_CONTENT, ""), - ], + create_setup_responses(false) + .into_iter() + .chain(create_launch_responses()) + .collect(), ); let exporter = InMemorySpanExporterBuilder::new().build(); let provider = SdkTracerProvider::builder() @@ -1929,7 +2085,6 @@ mod tests { "podman.prepare_images", "podman.prepare_storage", "podman.prepare_container", - "podman.start_container", ] { let child = spans .iter() @@ -2009,8 +2164,7 @@ mod tests { StatusCode::OK, r#"{"Id":"ctr-1","Name":"sandbox","State":{"Status":"exited","Running":false,"FinishedAt":"2026-08-12T16:39:13Z"},"Config":{}}"#, ), - StubResponse::new(StatusCode::NO_CONTENT, ""), - ], + ].into_iter().chain(restart_responses()).collect(), ); test_driver(start_socket.clone()) .start_sandbox("sandbox-1") @@ -2022,6 +2176,8 @@ mod tests { let (delete_socket, _requests, delete_handle) = spawn_podman_stub( "trace-delete", vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove channel if detached StubResponse::new(StatusCode::OK, "[]"), StubResponse::new(StatusCode::NO_CONTENT, ""), ], @@ -2892,6 +3048,8 @@ mod tests { let (socket_path, request_log, handle) = spawn_podman_stub( "delete-not-found", vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove channel if detached // list_containers returns empty (container already gone) StubResponse::new(StatusCode::OK, "[]"), // remove_volume @@ -2911,9 +3069,9 @@ mod tests { .lock() .expect("request log lock should not be poisoned") .clone(); - assert!(requests[0].contains("/libpod/containers/json")); + assert!(requests[2].contains("/libpod/containers/json")); assert_eq!( - requests[1], + requests[3], format!( "DELETE {}", api_path(&format!("/libpod/volumes/{volume_name}")) @@ -2961,6 +3119,159 @@ mod tests { ) } + fn restart_responses() -> Vec { + let mut archive = tar::Builder::new(Vec::new()); + let bundle = tar::Builder::new(Vec::new()).into_inner().unwrap(); + let mut header = tar::Header::new_gnu(); + header.set_size(bundle.len() as u64); + header.set_mode(0o600); + header.set_cksum(); + archive + .append_data(&mut header, "sandbox-bundle.tar", bundle.as_slice()) + .unwrap(); + vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor stop + StubResponse::new( + StatusCode::OK, + r#"{"Id":"supervisor","Name":"supervisor","State":{"Status":"exited","Running":false},"Config":{}}"#, + ), + StubResponse::new(StatusCode::OK, archive.into_inner().unwrap()), + StubResponse::new(StatusCode::OK, ""), // restore bootstrap + fence_response(), + StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start + StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor start + ] + } + + fn fence_response() -> StubResponse { + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct HostConfig { + network_mode: &'static str, + privileged: bool, + } + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Networks { + networks: std::collections::BTreeMap, + } + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Fence { + host_config: HostConfig, + network_settings: Networks, + } + StubResponse::new( + StatusCode::OK, + serde_json::to_vec(&Fence { + host_config: HostConfig { + network_mode: "none", + privileged: false, + }, + network_settings: Networks { + networks: std::collections::BTreeMap::default(), + }, + }) + .unwrap(), + ) + } + + fn created_response(id: &'static str) -> StubResponse { + #[derive(serde::Serialize)] + struct Created { + #[serde(rename = "Id")] + id: &'static str, + } + StubResponse::new( + StatusCode::CREATED, + serde_json::to_vec(&Created { id }).unwrap(), + ) + } + + fn image_response(id: &'static str) -> StubResponse { + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Config { + user: &'static str, + } + #[derive(serde::Serialize)] + #[serde(rename_all = "PascalCase")] + struct Image { + id: &'static str, + config: Config, + } + StubResponse::new( + StatusCode::OK, + serde_json::to_vec(&Image { + id, + config: Config { user: "1234:1235" }, + }) + .unwrap(), + ) + } + + fn create_setup_responses(proxy_secret: bool) -> Vec { + let mut responses = vec![ + StubResponse::new(StatusCode::OK, "{}"), // supervisor pull + StubResponse::new(StatusCode::OK, "{}"), // workload pull + image_response("sha256:sandbox"), + created_response("identity-reader"), + StubResponse::new(StatusCode::NOT_FOUND, ""), // reserved hierarchy absent + StubResponse::new(StatusCode::NOT_FOUND, ""), // optional passwd + StubResponse::new(StatusCode::NOT_FOUND, ""), // optional group + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove stopped reader + image_response("sha256:supervisor"), + StubResponse::new(StatusCode::CREATED, "{}"), // workspace volume + ]; + if proxy_secret { + responses.push(StubResponse::new(StatusCode::CREATED, "{}")); + } + responses.push(StubResponse::new(StatusCode::CREATED, "{}")); // channel volume + responses + } + + fn create_launch_responses() -> Vec { + vec![ + created_response("workload"), + fence_response(), + StubResponse::new(StatusCode::OK, ""), // workload archive + created_response("supervisor"), + StubResponse::new(StatusCode::OK, ""), // supervisor archive + StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start + StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor start + ] + } + + #[tokio::test] + async fn reserved_image_control_root_fails_before_workload_or_secrets() { + let (path, requests, handle) = spawn_podman_stub( + "reserved-control-root", + vec![ + StubResponse::new(StatusCode::OK, "{}"), + StubResponse::new(StatusCode::OK, "{}"), + image_response("sha256:image"), + created_response("identity-reader"), + StubResponse::new(StatusCode::OK, "existing reserved path"), + StubResponse::new(StatusCode::NO_CONTENT, ""), + ], + ); + let error = test_driver(path.clone()) + .create_sandbox(&plain_sandbox("id", "name")) + .await + .unwrap_err(); + assert!(error.to_string().contains("reserved /.openshell")); + handle.await.unwrap(); + assert!( + !requests + .lock() + .unwrap() + .iter() + .any(|request| request.contains("/libpod/volumes") + || request.contains("/libpod/secrets")) + ); + let _ = fs::remove_file(path); + } + #[tokio::test] async fn create_sandbox_removes_proxy_auth_secret_on_container_create_failure() { // A credential secret is staged before the container is created, so a @@ -2969,19 +3280,15 @@ mod tests { let auth_file = write_proxy_auth_file("create-fail"); let (socket_path, request_log, handle) = spawn_podman_stub( "create-container-fail", - vec![ - StubResponse::new(StatusCode::OK, "{}"), // pull supervisor image - StubResponse::new(StatusCode::OK, "{}"), // pull sandbox image - StubResponse::new( - StatusCode::OK, - r#"{"Id":"sha256:sandbox","Config":{"User":"1234:1235"}}"#, - ), // inspect sandbox image - StubResponse::new(StatusCode::CREATED, "{}"), // create volume - StubResponse::new(StatusCode::CREATED, "{}"), // create proxy-auth secret - StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, r#"{"message":"boom"}"#), // create container - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove volume - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove proxy-auth secret - ], + create_setup_responses(true) + .into_iter() + .chain([ + StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, "create failed"), + StubResponse::new(StatusCode::NO_CONTENT, ""), // channel + StubResponse::new(StatusCode::NO_CONTENT, ""), // workspace + StubResponse::new(StatusCode::NO_CONTENT, ""), // proxy secret + ]) + .collect(), ); let driver = test_driver_with_config(proxy_auth_config(socket_path.clone(), &auth_file)); @@ -3011,21 +3318,18 @@ mod tests { let auth_file = write_proxy_auth_file("start-fail"); let (socket_path, request_log, handle) = spawn_podman_stub( "create-start-fail", - vec![ - StubResponse::new(StatusCode::OK, "{}"), // pull supervisor image - StubResponse::new(StatusCode::OK, "{}"), // pull sandbox image - StubResponse::new( - StatusCode::OK, - r#"{"Id":"sha256:sandbox","Config":{"User":"1234:1235"}}"#, - ), // inspect sandbox image - StubResponse::new(StatusCode::CREATED, "{}"), // create volume - StubResponse::new(StatusCode::CREATED, "{}"), // create proxy-auth secret - StubResponse::new(StatusCode::CREATED, "{}"), // create container - StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, r#"{"message":"boom"}"#), // start container - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove container - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove volume - StubResponse::new(StatusCode::NO_CONTENT, ""), // cleanup: remove proxy-auth secret - ], + create_setup_responses(true) + .into_iter() + .chain(create_launch_responses().into_iter().take(6)) + .chain([ + StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, "supervisor start failed"), + StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor + StubResponse::new(StatusCode::NO_CONTENT, ""), // workload + StubResponse::new(StatusCode::NO_CONTENT, ""), // channel + StubResponse::new(StatusCode::NO_CONTENT, ""), // workspace + StubResponse::new(StatusCode::NO_CONTENT, ""), // proxy secret + ]) + .collect(), ); let driver = test_driver_with_config(proxy_auth_config(socket_path.clone(), &auth_file)); @@ -3055,7 +3359,9 @@ mod tests { let (socket_path, request_log, handle) = spawn_podman_stub( "delete-proxy-auth", vec![ - StubResponse::new(StatusCode::OK, "[]"), // list_containers (not found) + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove channel if detached + StubResponse::new(StatusCode::OK, "[]"), // list_containers (not found) StubResponse::new(StatusCode::NO_CONTENT, ""), // remove volume StubResponse::new(StatusCode::NO_CONTENT, ""), // remove token secret StubResponse::new(StatusCode::NO_CONTENT, ""), // remove proxy-auth secret @@ -3098,10 +3404,14 @@ mod tests { let (socket_path, request_log, handle) = spawn_podman_stub( "delete-label-lookup", vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove channel if detached // list_containers by label StubResponse::new(StatusCode::OK, list_body), // single timed remove_container operation StubResponse::new(StatusCode::NO_CONTENT, ""), + // channel volume, now detached + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove_volume StubResponse::new(StatusCode::NO_CONTENT, ""), ], @@ -3119,9 +3429,9 @@ mod tests { .lock() .expect("request log lock should not be poisoned") .clone(); - assert!(requests[0].contains("/libpod/containers/json")); + assert!(requests[2].contains("/libpod/containers/json")); assert_eq!( - requests[1], + requests[3], format!( "DELETE {}", api_path(&format!( @@ -3130,7 +3440,7 @@ mod tests { ) ); assert_eq!( - requests[2], + requests[5], format!( "DELETE {}", api_path(&format!("/libpod/volumes/{volume_name}")) diff --git a/crates/openshell-driver-podman/src/grpc.rs b/crates/openshell-driver-podman/src/grpc.rs index fedeea3068..d18ad5d88d 100644 --- a/crates/openshell-driver-podman/src/grpc.rs +++ b/crates/openshell-driver-podman/src/grpc.rs @@ -162,8 +162,7 @@ impl ComputeDriver for ComputeDriverService { .into_inner() .sandbox .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; - self.driver - .create_sandbox(&sandbox) + Box::pin(self.driver.create_sandbox(&sandbox)) .await .map_err(Status::from)?; Ok(Response::new(CreateSandboxResponse {})) @@ -680,6 +679,8 @@ mod tests { let (socket_path, request_log, handle) = spawn_podman_stub( "forward-id", vec![ + StubResponse::new(StatusCode::NO_CONTENT, ""), // companion + StubResponse::new(StatusCode::NO_CONTENT, ""), // channel // list_containers returns empty (container already gone) StubResponse::new(StatusCode::OK, "[]"), // remove_volume @@ -708,9 +709,9 @@ mod tests { .lock() .expect("request log lock should not be poisoned") .clone(); - assert!(requests[0].contains("/libpod/containers/json")); + assert!(requests[2].contains("/libpod/containers/json")); assert_eq!( - requests[1], + requests[3], format!( "DELETE {}", api_path(&format!("/libpod/volumes/{volume_name}")) diff --git a/crates/openshell-driver-podman/src/isolation.rs b/crates/openshell-driver-podman/src/isolation.rs new file mode 100644 index 0000000000..9a33e6559f --- /dev/null +++ b/crates/openshell-driver-podman/src/isolation.rs @@ -0,0 +1,381 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Podman-owned provisioning for the common authenticated isolation channel. + +use std::collections::{BTreeMap, HashMap}; +use std::path::PathBuf; + +use openshell_core::ComputeDriverError; +use openshell_core::proto::compute::v1::DriverSandbox; +use openshell_isolation_interface::boundary_protocol::{ + BoundaryClientTls, BoundaryConfig, BoundaryListener, BoundaryServerTls, BoundaryTopology, + BoundaryTransport, generate_boundary_mutual_tls_material, +}; +use openshell_isolation_interface::contract::{DriverFenceEvidence, ResolvedWorkloadIdentity}; + +pub const LABEL_ROLE: &str = "openshell.io/isolation-role"; +pub const WORKLOAD_FILTER: &str = "openshell.io/isolation-role=sandbox"; +pub const CHANNEL_ROOT: &str = "/.openshell/channel"; +pub const BOOTSTRAP_PATH: &str = "/.openshell/channel/sandbox/bootstrap.json"; +pub const TOPOLOGY_PATH: &str = "/.openshell/supervisor/topology.payload"; +pub const RESTART_BUNDLE_PATH: &str = "/.openshell/supervisor/sandbox-bundle.tar"; +const SOCKET_PATH: &str = "/.openshell/channel/sandbox/control.sock"; + +pub fn supervisor_name(id: &str) -> String { + format!("openshell-supervisor-{id}") +} +pub fn channel_volume_name(id: &str) -> String { + format!("openshell-channel-{id}") +} + +fn invalid(error: impl std::fmt::Display) -> ComputeDriverError { + ComputeDriverError::Precondition(error.to_string()) +} + +/// Resolve policy names against the pinned workload image, never the gateway. +pub fn resolve_identity( + sandbox: &DriverSandbox, + image_id: &str, + image_user: &str, + passwd: &[u8], + group: &[u8], +) -> Result { + let passwd = std::str::from_utf8(passwd).map_err(invalid)?; + let group = std::str::from_utf8(group).map_err(invalid)?; + let accounts: Vec<_> = passwd + .lines() + .filter_map(|line| { + let mut fields = line.split(':'); + let name = fields.next()?; + fields.next()?; + Some(( + name, + fields.next()?.parse::().ok()?, + fields.next()?.parse::().ok()?, + )) + }) + .collect(); + let groups: Vec<_> = group + .lines() + .filter_map(|line| { + let mut fields = line.split(':'); + let name = fields.next()?; + fields.next()?; + Some((name, fields.next()?.parse::().ok()?, fields.next()?)) + }) + .collect(); + let request = sandbox + .spec + .as_ref() + .and_then(|spec| spec.workload_identity.as_ref()); + let requested_user = request.map_or("", |identity| identity.user.trim()); + let requested_group = request.map_or("", |identity| identity.group.trim()); + let (image_user, image_group) = image_user.split_once(':').unwrap_or((image_user, "")); + let user = if requested_user.is_empty() { + image_user + } else { + requested_user + }; + let group = if requested_group.is_empty() { + image_group + } else { + requested_group + }; + let account = accounts + .iter() + .find(|(name, uid, _)| *name == user || user.parse::().ok() == Some(*uid)); + let uid = user + .parse() + .ok() + .or_else(|| account.map(|(_, uid, _)| *uid)) + .ok_or_else(|| invalid("configure a non-root workload user present in the pinned image"))?; + let gid = if group.is_empty() { + account.map(|(_, _, gid)| *gid) + } else { + group.parse().ok().or_else(|| { + groups + .iter() + .find(|(name, _, _)| *name == group) + .map(|(_, gid, _)| *gid) + }) + } + .ok_or_else(|| { + invalid("configure an explicit workload group for a UID without an image passwd entry") + })?; + let supplemental = account.map_or_else(Vec::new, |(username, _, _)| { + groups + .iter() + .filter(|(_, id, members)| { + *id != gid && members.split(',').any(|member| member == *username) + }) + .map(|(_, gid, _)| *gid) + .collect() + }); + let source = if requested_user.is_empty() && requested_group.is_empty() { + "image" + } else { + "policy" + }; + ResolvedWorkloadIdentity::new(uid, gid, supplemental, source.into(), image_id.into()) + .map_err(invalid) +} + +pub struct BootstrapArchives { + pub workload: Vec, + pub supervisor: Vec, +} + +/// The shared volume contains only sandbox credentials. Supervisor credentials, +/// gateway authorization, and the restart copy never enter that volume. +pub fn bootstrap_archives( + sandbox_id: &str, + container_id: &str, + identity: &ResolvedWorkloadIdentity, + child_env: HashMap, +) -> Result { + let tls = generate_boundary_mutual_tls_material().map_err(invalid)?; + let resource_claims = BTreeMap::from([ + ("podman.container_id".into(), container_id.into()), + ( + "podman.image_identity".into(), + identity.resource_digest.clone(), + ), + ]); + let driver_fence = DriverFenceEvidence::Podman { + container_id: container_id.into(), + network_mode: "none".into(), + unexpected_networks: Vec::new(), + }; + let generation = uuid::Uuid::new_v4().to_string(); + let session_epoch = uuid::Uuid::new_v4().to_string(); + let bootstrap_token = format!( + "{}{}", + uuid::Uuid::new_v4().simple(), + uuid::Uuid::new_v4().simple() + ); + let config = BoundaryConfig { + boundary_id: sandbox_id.into(), + generation: generation.clone(), + session_epoch: session_epoch.clone(), + bootstrap_token: bootstrap_token.clone(), + listener: BoundaryListener::Unix { + socket_path: PathBuf::from(SOCKET_PATH), + tls: BoundaryServerTls { + certificate_chain_path: PathBuf::from("/.openshell/channel/sandbox/server.crt"), + private_key_path: PathBuf::from("/.openshell/channel/sandbox/server.key"), + client_ca_certificate_path: PathBuf::from( + "/.openshell/channel/sandbox/client-ca.crt", + ), + }, + }, + resource_claims: resource_claims.clone(), + resource_claim_files: BTreeMap::new(), + workload_identity: identity.clone(), + driver_fence: driver_fence.clone(), + child_env, + }; + let topology = BoundaryTopology { + boundary_id: sandbox_id.into(), + generation, + session_epoch, + bootstrap_token, + transport: BoundaryTransport::Unix { + socket_path: PathBuf::from(SOCKET_PATH), + tls: BoundaryClientTls { + server_name: tls.server_name, + ca_certificate_pem: tls.ca_certificate_pem.clone(), + certificate_chain_pem: tls.supervisor_certificate_pem, + private_key_pem: tls.supervisor_private_key_pem, + }, + }, + host_gateway_ip: None, + resource_claims, + workload_identity: identity.clone(), + driver_fence, + }; + let mut workload = Archive::new(identity); + workload.directory(".openshell", 0o755, false)?; + workload.directory(".openshell/channel", 0o755, false)?; + workload.directory(".openshell/channel/sandbox", 0o711, true)?; + workload.directory("sandbox", 0o700, true)?; + workload.file( + BOOTSTRAP_PATH, + &serde_json::to_vec(&config).map_err(invalid)?, + )?; + workload.file( + "/.openshell/channel/sandbox/server.crt", + tls.sandbox_certificate_pem.as_bytes(), + )?; + workload.file( + "/.openshell/channel/sandbox/server.key", + tls.sandbox_private_key_pem.as_bytes(), + )?; + workload.file( + "/.openshell/channel/sandbox/client-ca.crt", + tls.ca_certificate_pem.as_bytes(), + )?; + let workload = workload.finish()?; + let mut supervisor = Archive::new(identity); + supervisor.directory(".openshell", 0o755, false)?; + supervisor.directory(".openshell/supervisor", 0o700, true)?; + supervisor.file( + TOPOLOGY_PATH, + &serde_json::to_vec(&topology).map_err(invalid)?, + )?; + supervisor.file(RESTART_BUNDLE_PATH, &workload)?; + Ok(BootstrapArchives { + workload, + supervisor: supervisor.finish()?, + }) +} + +struct Archive<'a> { + builder: tar::Builder>, + identity: &'a ResolvedWorkloadIdentity, +} +impl<'a> Archive<'a> { + fn new(identity: &'a ResolvedWorkloadIdentity) -> Self { + Self { + builder: tar::Builder::new(Vec::new()), + identity, + } + } + fn directory(&mut self, path: &str, mode: u32, owned: bool) -> Result<(), ComputeDriverError> { + self.append(path, mode, owned, tar::EntryType::Directory, &[]) + } + fn file(&mut self, path: &str, content: &[u8]) -> Result<(), ComputeDriverError> { + self.append( + path.trim_start_matches('/'), + 0o600, + true, + tar::EntryType::Regular, + content, + ) + } + fn append( + &mut self, + path: &str, + mode: u32, + owned: bool, + kind: tar::EntryType, + content: &[u8], + ) -> Result<(), ComputeDriverError> { + let mut header = tar::Header::new_gnu(); + header.set_entry_type(kind); + header.set_mode(mode); + header.set_uid(if owned { + u64::from(self.identity.uid) + } else { + 0 + }); + header.set_gid(if owned { + u64::from(self.identity.gid) + } else { + 0 + }); + header.set_size(content.len() as u64); + header.set_mtime(0); + header.set_cksum(); + self.builder + .append_data(&mut header, path, content) + .map_err(invalid) + } + fn finish(self) -> Result, ComputeDriverError> { + self.builder.into_inner().map_err(invalid) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Read as _; + + #[test] + fn identity_uses_pinned_image_accounts_and_rejects_root() { + let sandbox = DriverSandbox::default(); + let passwd = b"root:x:0:0:root:/root:/bin/sh\nagent:x:1000:1001::/home/agent:/bin/sh\n"; + let groups = b"agent:x:1001:\ndata:x:2000:agent\n"; + let identity = + resolve_identity(&sandbox, "sha256:pinned", "agent", passwd, groups).unwrap(); + assert_eq!((identity.uid, identity.gid), (1000, 1001)); + assert_eq!(identity.supplementary_gids, vec![2000]); + assert_eq!(identity.resource_digest, "sha256:pinned"); + assert!(resolve_identity(&sandbox, "sha256:pinned", "root", passwd, groups).is_err()); + assert!(resolve_identity(&sandbox, "sha256:pinned", "", passwd, groups).is_err()); + assert!(resolve_identity(&sandbox, "sha256:pinned", "2000", passwd, groups).is_err()); + } + + fn files(bytes: &[u8]) -> BTreeMap> { + tar::Archive::new(bytes) + .entries() + .unwrap() + .filter_map(|entry| { + let mut entry = entry.unwrap(); + if !entry.header().entry_type().is_file() { + return None; + } + let path = entry.path().unwrap().into_owned(); + assert_eq!(entry.header().mode().unwrap(), 0o600); + assert_eq!(entry.header().uid().unwrap(), 1000); + let mut content = Vec::new(); + entry.read_to_end(&mut content).unwrap(); + Some((path, content)) + }) + .collect() + } + + #[test] + fn archives_separate_supervisor_credentials_and_bind_one_channel() { + let identity = ResolvedWorkloadIdentity::new( + 1000, + 1001, + vec![], + "image".into(), + "sha256:image".into(), + ) + .unwrap(); + let archives = + bootstrap_archives("sandbox", "container", &identity, HashMap::new()).unwrap(); + let workload = files(&archives.workload); + let supervisor = files(&archives.supervisor); + assert_eq!(workload.len(), 4); + assert_eq!(supervisor.len(), 2); + assert!( + workload + .keys() + .all(|path| path.starts_with(".openshell/channel/sandbox")) + ); + assert!( + supervisor + .keys() + .all(|path| path.starts_with(".openshell/supervisor")) + ); + let config: BoundaryConfig = serde_json::from_slice( + workload + .get(&PathBuf::from(BOOTSTRAP_PATH.trim_start_matches('/'))) + .unwrap(), + ) + .unwrap(); + let topology: BoundaryTopology = serde_json::from_slice( + supervisor + .get(&PathBuf::from(TOPOLOGY_PATH.trim_start_matches('/'))) + .unwrap(), + ) + .unwrap(); + assert_eq!(config.boundary_id, topology.boundary_id); + assert_eq!(config.bootstrap_token, topology.bootstrap_token); + assert_eq!(config.driver_fence, topology.driver_fence); + assert_eq!(config.workload_identity, identity); + topology + .driver_fence + .validate_for_backend("podman") + .unwrap(); + assert_eq!( + supervisor + .get(&PathBuf::from(RESTART_BUNDLE_PATH.trim_start_matches('/'))) + .unwrap(), + &archives.workload + ); + } +} diff --git a/crates/openshell-driver-podman/src/lib.rs b/crates/openshell-driver-podman/src/lib.rs index 115e64eb2f..fa06cf4864 100644 --- a/crates/openshell-driver-podman/src/lib.rs +++ b/crates/openshell-driver-podman/src/lib.rs @@ -6,6 +6,7 @@ pub mod config; pub(crate) mod container; pub mod driver; pub mod grpc; +mod isolation; pub mod otel_tracing; mod socket_discovery; #[cfg(test)] diff --git a/crates/openshell-driver-podman/src/test_utils.rs b/crates/openshell-driver-podman/src/test_utils.rs index ec5c8f7f11..24e9d4d8ed 100644 --- a/crates/openshell-driver-podman/src/test_utils.rs +++ b/crates/openshell-driver-podman/src/test_utils.rs @@ -20,12 +20,12 @@ use tokio::net::UnixListener; #[derive(Clone)] pub struct StubResponse { pub status: StatusCode, - pub body: String, + pub body: Bytes, pub delay: Duration, } impl StubResponse { - pub fn new(status: StatusCode, body: impl Into) -> Self { + pub fn new(status: StatusCode, body: impl Into) -> Self { Self { status, body: body.into(), @@ -108,7 +108,7 @@ pub fn spawn_podman_stub( Ok::<_, Infallible>( hyper::Response::builder() .status(response.status) - .body(Full::new(Bytes::from(response.body))) + .body(Full::new(response.body)) .expect("stub response should build"), ) } diff --git a/crates/openshell-driver-podman/src/watcher.rs b/crates/openshell-driver-podman/src/watcher.rs index b99ecdd540..6e5eb52f27 100644 --- a/crates/openshell-driver-podman/src/watcher.rs +++ b/crates/openshell-driver-podman/src/watcher.rs @@ -139,14 +139,16 @@ pub async fn start_watch( let mut event_rx = client.events_stream(LABEL_MANAGED_FILTER).await?; // 2. List existing containers for initial state sync. - let existing = client.list_containers(&[LABEL_MANAGED_FILTER]).await?; + let existing = client + .list_containers(&[LABEL_MANAGED_FILTER, crate::isolation::WORKLOAD_FILTER]) + .await?; for entry in &existing { // For running containers, use inspect to get full state including // health check status — matching the same condition derivation used // for live events. if entry.state == "running" { - match client.inspect_container(&entry.id).await { + match inspect_workload(&client, &entry.id).await { Ok(inspect) => { if let Some(sandbox) = driver_sandbox_from_inspect(&inspect) { if tx.send(Ok(sandbox_event(sandbox))).await.is_err() { @@ -246,11 +248,34 @@ async fn map_podman_event( return None; } + if event + .actor + .attributes + .get(crate::isolation::LABEL_ROLE) + .is_some_and(|role| role == "supervisor") + { + let id_filter = format!("{LABEL_SANDBOX_ID}={sandbox_id}"); + let workloads = client + .list_containers(&[ + LABEL_MANAGED_FILTER, + &id_filter, + crate::isolation::WORKLOAD_FILTER, + ]) + .await + .ok()?; + let workload = workloads.first()?; + return inspect_workload(client, &workload.id) + .await + .ok() + .and_then(|inspect| driver_sandbox_from_inspect(&inspect)) + .map(sandbox_event); + } + match event.action.as_str() { "remove" => Some(deleted_event(sandbox_id.clone())), "create" | "start" | "stop" | "die" | "health_status" => { // Inspect the container to get current state. - match client.inspect_container(container_id).await { + match inspect_workload(client, container_id).await { Ok(inspect) => { if lifecycle_event_fences.matches_previous_exit( event, @@ -327,6 +352,54 @@ async fn map_podman_event( } } +/// A workload is ready only when its independent supervisor is healthy. This +/// check runs both on watch reconciliation and on events, and contains a lost +/// supervisor even when the gateway missed the original exit event. +pub async fn inspect_workload( + client: &PodmanClient, + id: &str, +) -> Result { + let mut workload = client.inspect_container(id).await?; + if workload + .config + .labels + .get(crate::isolation::LABEL_ROLE) + .is_none_or(|role| role != "sandbox") + { + return Ok(workload); + } + let Some(sandbox_id) = workload.config.labels.get(LABEL_SANDBOX_ID) else { + return Ok(workload); + }; + let supervisor = client + .inspect_container(&crate::isolation::supervisor_name(sandbox_id)) + .await; + if workload.state.running { + match supervisor { + Ok(supervisor) if supervisor.state.running => { + workload.state.health = supervisor.state.health; + } + Ok(supervisor) + if supervisor.state.status == "configured" + || supervisor.state.status == "created" => + { + workload.state.health = Some(HealthState { + status: "starting".into(), + }); + } + // Both containers exist before initial start. A missing or exited + // companion therefore requires containment, including after a + // gateway restart that missed the original Podman exit event. + Ok(_) | Err(PodmanApiError::NotFound(_)) => { + client.stop_container(&workload.id, 0).await?; + workload = client.inspect_container(&workload.id).await?; + } + Err(error) => return Err(error), + } + } + Ok(workload) +} + /// Construct a `DriverSandbox` from common fields. /// /// Centralises the boilerplate that every event/inspect/list path shares: @@ -510,6 +583,65 @@ fn condition_from_state(state: &ContainerState) -> DriverCondition { mod tests { use super::*; + #[tokio::test] + async fn missing_supervisor_stops_workload_during_reconciliation() { + use crate::test_utils::{StubResponse, spawn_podman_stub}; + use hyper::StatusCode; + let (path, requests, handle) = spawn_podman_stub( + "lost-supervisor", + vec![ + StubResponse::new( + StatusCode::OK, + r#"{"Id":"workload","Name":"workload","State":{"Status":"running","Running":true},"Config":{"Labels":{"openshell.ai/sandbox-id":"test","openshell.io/isolation-role":"sandbox"}}}"#, + ), + StubResponse::new(StatusCode::NOT_FOUND, "missing companion"), + StubResponse::new(StatusCode::NO_CONTENT, ""), + StubResponse::new( + StatusCode::OK, + r#"{"Id":"workload","Name":"workload","State":{"Status":"exited","Running":false},"Config":{}}"#, + ), + ], + ); + let client = PodmanClient::new(path.clone()); + let inspected = inspect_workload(&client, "workload").await.unwrap(); + assert!(!inspected.state.running); + handle.await.unwrap(); + assert!( + requests + .lock() + .unwrap() + .iter() + .any(|request| request.ends_with("/libpod/containers/workload/stop?timeout=0")) + ); + let _ = std::fs::remove_file(path); + } + + #[tokio::test] + async fn created_supervisor_keeps_bootstrapping_workload_starting() { + use crate::test_utils::{StubResponse, spawn_podman_stub}; + use hyper::StatusCode; + let (path, requests, handle) = spawn_podman_stub( + "starting-supervisor", + vec![ + StubResponse::new( + StatusCode::OK, + r#"{"Id":"workload","Name":"workload","State":{"Status":"running","Running":true},"Config":{"Labels":{"openshell.ai/sandbox-id":"test","openshell.io/isolation-role":"sandbox"}}}"#, + ), + StubResponse::new( + StatusCode::OK, + r#"{"Id":"supervisor","Name":"supervisor","State":{"Status":"configured","Running":false},"Config":{}}"#, + ), + ], + ); + let client = PodmanClient::new(path.clone()); + let inspected = inspect_workload(&client, "workload").await.unwrap(); + assert!(inspected.state.running); + assert_eq!(inspected.state.health.unwrap().status, "starting"); + handle.await.unwrap(); + assert_eq!(requests.lock().unwrap().len(), 2); + let _ = std::fs::remove_file(path); + } + fn podman_event(action: &str, sandbox_id: &str, time_nano: i64) -> PodmanEvent { PodmanEvent { event_type: "container".to_string(), diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index ed9b38383d..e23137827e 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -22,28 +22,26 @@ Gateway CLI flag > gateway OPENSHELL_* env var > TOML file > built-in defa ## Package-Managed Locations -Package-managed gateways use either built-in defaults or a package-seeded TOML file. Set `OPENSHELL_GATEWAY_CONFIG` in the launch environment to use a different file. +Package-managed gateways do not require a TOML file. Create one at the package's optional config location when you need to override built-in defaults. Set `OPENSHELL_GATEWAY_CONFIG` in the launch environment to use a different file. -| Package | Gateway TOML location | +| Package | Optional Gateway TOML location | |---|---| | Homebrew | `$XDG_CONFIG_HOME/openshell/gateway.toml` when it exists, otherwise the Homebrew prefix config such as `/opt/homebrew/var/openshell/gateway.toml`. | | Debian/Ubuntu | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml` for the systemd user service. | -| Fedora/RHEL RPM | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml`; the systemd user service seeds this file from the packaged template on first start. | +| Fedora/RHEL RPM | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml` for the systemd user service. | | Snap | `$SNAP_COMMON/gateway.toml`, usually `/var/snap/openshell/common/gateway.toml`. | The Fedora/RHEL RPM template leaves `[openshell.gateway].bind_address` unset. The gateway therefore uses its built-in `127.0.0.1:17670` primary listener. The Podman driver negotiates separate, restricted listeners for sandbox callbacks, so the primary listener does not need a wildcard address. Set `bind_address` explicitly only when clients must reach the primary multiplexed API through another interface. -The Homebrew formula creates its prefix config without setting `bind_address`, so the gateway uses its built-in `127.0.0.1:17670` primary listener. Docker Desktop and Podman Machine reuse that listener for sandbox callbacks. A user config takes precedence. - -Homebrew and RPM upgrades migrate only exact package-generated schema-v1 defaults. Homebrew recognizes both its empty v1 prefix config and the affected IPv6-loopback variant. RPM recognizes the v1 file seeded by its systemd user service. Package upgrades never rewrite an edited file; migrate an edited v1 file manually with the steps below. +The Homebrew formula creates its prefix config without setting `bind_address`, so the gateway uses its built-in `127.0.0.1:17670` primary listener. Docker Desktop and Podman Machine reuse that listener for sandbox callbacks. A user config takes precedence. Upgrades preserve user-edited configs and migrate only an unchanged prefix config generated with the affected IPv6-loopback default. ## Layout -The file is rooted at `[openshell]`. Gateway-wide settings live under `[openshell.gateway]`. Each compute driver owns its own `[openshell.drivers.]` table. Credential drivers own `[openshell.credential_drivers.]` tables. Driver-specific values are never inherited from gateway scope. +The file is rooted at `[openshell]`. Gateway-wide settings live under `[openshell.gateway]`. Each compute driver owns its own `[openshell.drivers.]` table. Credential drivers own `[openshell.credential_drivers.]` tables. Shared compute-driver keys set at gateway scope are inherited into compute driver tables when not overridden. ```toml [openshell] -version = 2 +version = 1 [openshell.gateway] # ... gateway-wide settings ... @@ -116,7 +114,7 @@ A complete gateway configuration covering every section. Trim to the fields you # SPDX-License-Identifier: Apache-2.0 [openshell] -version = 2 +version = 1 [openshell.gateway] name = "production-us-west" @@ -126,14 +124,15 @@ metrics_bind_address = "0.0.0.0:9090" log_level = "info" -# When omitted, the gateway auto-detects Kubernetes, then Podman, then Docker. +# When empty, the gateway auto-detects Kubernetes, then Podman, then Docker. # VM is never auto-detected and requires an explicit entry here. -compute_driver = "kubernetes" +compute_drivers = ["kubernetes"] # Optional external provider credential storage backend. Omit this key to use # the gateway's default encrypted database credential storage. credential_drivers = ["kubernetes-secrets"] +sandbox_namespace = "openshell" ssh_session_ttl_secs = 3600 # Reject invalid policy generations securely by default. Set @@ -150,11 +149,18 @@ enable_loopback_service_http = true # Set true only for local plaintext gateways or trusted TLS termination. disable_tls = false -# Guest TLS paths remain gateway settings. TLS-enabled Docker, Podman, and VM -# gateways require a complete bundle unless package-managed local TLS supplies -# it automatically. Omit all three when TLS is disabled. Kubernetes projects -# sandbox TLS from client_tls_secret_name instead. Driver tables must not repeat -# these fields. +# Shared driver defaults. These inherit into [openshell.drivers.] tables +# when the driver-specific table does not override them. +default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +# Defaults to the gateway version; override to pin a specific build. +sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:" +# Defaults to the gateway version; override to pin a specific build. +# supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" +client_tls_secret_name = "openshell-client-tls" +service_account_name = "openshell-sandbox" +host_gateway_ip = "10.0.0.1" +enable_user_namespaces = false +sa_token_ttl_secs = 3600 guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" guest_tls_key = "/etc/openshell/certs/client-key.pem" @@ -188,6 +194,7 @@ timeout = "500ms" cert_path = "/etc/openshell/certs/gateway.pem" key_path = "/etc/openshell/certs/gateway-key.pem" client_ca_path = "/etc/openshell/certs/client-ca.pem" +require_client_auth = false # Optional: SNI-based dual certificate for external (e.g. ACME) TLS. # external_cert_path = "/etc/openshell/certs/external.pem" # external_key_path = "/etc/openshell/certs/external-key.pem" @@ -198,7 +205,7 @@ signing_key_path = "/etc/openshell/jwt/signing.pem" public_key_path = "/etc/openshell/jwt/public.pem" kid_path = "/etc/openshell/jwt/kid" gateway_id = "openshell" -# Omit only for local single-player Docker, Podman, or VM gateways. +# Omit or set to 0 only for local single-player Docker, Podman, or VM gateways. ttl_secs = 3600 [openshell.gateway.auth] @@ -241,33 +248,18 @@ failure_policy = "fail_closed" rpc = "openshell.v1.OpenShell/UpdateConfig" phases = ["validate"] -[openshell.drivers.kubernetes] -namespace = "openshell" -# Required in raw TOML; Helm derives this from the gateway Service. -grpc_endpoint = "https://openshell-gateway.openshell.svc:8080" -default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -# Defaults to the gateway version; override to pin a specific build. -# supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -client_tls_secret_name = "openshell-client-tls" -service_account_name = "openshell-sandbox" -host_gateway_ip = "10.0.0.1" -enable_user_namespaces = false -sa_token_ttl_secs = 3600 - [openshell.credential_drivers.kubernetes-secrets] namespace = "openshell" allow_reference_namespace = false ``` -Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth] enabled = true` to map a verified client certificate to a CLI user identity. This application-layer identity switch does not control the TLS handshake. When `client_ca_path` is set without OIDC, the listener requires a valid client certificate. When OIDC is configured, bearer-only clients may connect; the listener still validates any client certificate they present against the configured CA. Kubernetes deployments must leave `mtls_auth.enabled` unset and use OIDC or a trusted access proxy; the Helm chart does not render this table. - -The client-certificate handshake policy is derived and has no `require_client_auth` TOML field. This preserves bearer-only OIDC clients and prevents a file setting from silently weakening CA-only gateways. +Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth] enabled = true` to authenticate CLI callers from verified client certificates. Kubernetes deployments must leave this unset and use OIDC or a trusted access proxy; the Helm chart does not render this table. `[openshell.gateway.tls]` supports optional SNI-based dual-certificate mode for deployments that need separate internal and external server certificates. Set `external_cert_path` and `external_key_path` to point at the external (e.g. ACME/publicly-trusted) certificate and key. List the hostnames that should be served with the external certificate in `external_server_names`. Connections whose TLS SNI hostname matches one of those names receive the external certificate; all other connections (including those with no SNI) receive the primary internal certificate from `cert_path`/`key_path`. Both fields must be set together — providing only one is a configuration error. On Kubernetes with the Helm chart, the external certificate is managed automatically when `certManager.serverIssuerRef.name` is set; the chart populates these fields from the cert-manager-issued external server certificate. `[openshell.gateway] policy_validation_failure_mode` controls what sandbox supervisors do when a complete candidate policy fails runtime validation. The default, `fail_closed`, deactivates the previous network policy, closes relays pinned to it, and denies new egress until a valid generation loads. `retain_last_valid` leaves the previous valid generation active. Both modes reject the candidate atomically; startup always fails closed when no previous valid generation exists. Gateway mutation paths that can preflight a known effective scope reject invalid candidates before persistence and leave the active policy unchanged regardless of this setting. Changing the value requires restarting the gateway so it can reload `gateway.toml` and distribute the new posture to sandbox supervisors. -`[openshell.gateway.gateway_jwt] ttl_secs` controls gateway-minted sandbox JWT lifetime. Omit it for a non-expiring token: the token `exp` claim and `expires_at_ms` response field become `0`. Use this only for local single-player Docker, Podman, or VM gateways. Explicit `0` is invalid. Kubernetes and other shared deployments should set a positive TTL; Helm renders `3600` seconds by default, and the gateway logs a warning when a Kubernetes gateway omits the field. +`[openshell.gateway.gateway_jwt] ttl_secs` controls gateway-minted sandbox JWT lifetime. It defaults to `3600` seconds and must be between `60` and `3600` seconds so sandbox sessions can rotate short-lived credentials safely. `[openshell.gateway.auth] allow_unauthenticated_users = true` is an unsafe local-development and trusted-proxy escape hatch. It accepts user-facing CLI/API calls without OIDC or mTLS credentials while sandbox supervisors still authenticate with gateway-minted sandbox JWTs. Leave it false for shared and production gateways. @@ -411,7 +403,7 @@ The gateway validates snapshot structure and provider-profile semantics. It trea `failure_policy` accepts `fail_closed` or `fail_open`. `timeout` accepts `ms` and `s` suffixes. In `dynamic` mode, binding overrides may select a manifest binding by `id`, `rpc`, or `service` plus `method`; they can disable a binding, narrow its phases, or override its failure policy. -`image_pull_policy` is a shared driver setting with the canonical values `always`, `if_not_present`, `never`, and `newer`. Set it inside the relevant driver table. Drivers translate these values to their runtime APIs; `newer` is supported only by Podman and is rejected at Docker and Kubernetes startup. +`image_pull_policy` is intentionally not a shared gateway key. Kubernetes and Docker use `Always`, `IfNotPresent`, or `Never`. Podman uses `always`, `missing`, `never`, or `newer`. Set it inside the relevant driver table. ## Credential Drivers @@ -500,9 +492,7 @@ args = [ ## Driver References -Each example is a complete TOML file for one compute driver. The examples repeat `[openshell]` and `[openshell.gateway]` so they stay copyable, and the driver tables list the accepted driver-specific keys. Drivers receive only their own tables, and the gateway rejects unknown gateway and driver fields. - -Kubernetes configurations set `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`. Docker configurations use `sandbox_label`; the legacy `sandbox_namespace` key is rejected. +Each example is a complete TOML file for one compute driver. The examples repeat `[openshell]` and `[openshell.gateway]` so they stay copyable, and the driver tables list the accepted driver-specific keys. Driver-specific values override inherited gateway defaults. The gateway rejects unknown driver fields after inheritance is merged. ### Kubernetes @@ -510,14 +500,14 @@ The gateway runs as a Pod and creates sandbox Pods in another namespace. mTLS ma ```toml [openshell] -version = 2 +version = 1 [openshell.gateway] bind_address = "0.0.0.0:8080" health_bind_address = "0.0.0.0:8081" metrics_bind_address = "0.0.0.0:9090" log_level = "info" -compute_driver = "kubernetes" +compute_drivers = ["kubernetes"] [openshell.gateway.tls] cert_path = "/etc/openshell-tls/server/tls.crt" @@ -540,11 +530,11 @@ workspace_mode = "shared" namespace = "agents" service_account_name = "openshell-sandbox" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "if_not_present" +image_pull_policy = "IfNotPresent" image_pull_secrets = ["regcred"] # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -supervisor_image_pull_policy = "if_not_present" +supervisor_image_pull_policy = "IfNotPresent" # Optional corporate HTTP forward proxy for policy-approved TLS egress. The # sandbox workload cannot select or override these settings. Only http:// proxy @@ -570,10 +560,7 @@ supervisor_image_pull_policy = "if_not_present" # Last resort for hostname-filtering proxy ACLs. The proxy resolves the target, # so its ACL becomes part of the egress boundary for proxied connections. # proxy_connect_by_hostname = true -# Required in raw gateway TOML because `namespace` identifies sandbox -# placement, not the gateway Service. Helm renders this from the release's -# gateway Service name and namespace. -grpc_endpoint = "https://openshell-gateway.openshell.svc:8080" +grpc_endpoint = "https://openshell-gateway.agents.svc:8080" ssh_socket_path = "/run/openshell/ssh.sock" client_tls_secret_name = "openshell-client-tls" host_gateway_ip = "10.0.0.1" @@ -633,56 +620,18 @@ The gateway verifies supervisor JWT-SVIDs with JWT bundles fetched from the SPIFFE Workload API, so this validation path does not require gateway access to the SPIRE OIDC discovery endpoint or its TLS CA. -### MXC - -The MXC driver runs Windows workloads through `wxc-exec`. Enable ETW auditing to -map Windows Sandboxing provider events into the gateway's OCSF stream. - -```toml -[openshell] -version = 1 - -[openshell.gateway] -bind_address = "127.0.0.1:17670" -log_level = "info" -compute_drivers = ["mxc"] - -[openshell.drivers.mxc] -wxc_exec_path = "C:\\mxc-kit\\bin\\wxc-exec.exe" -backend = "process_container" -default_configuration_id = "composable" -pc_least_privilege = false -pc_capabilities = [] -debug = false -etw_audit = true -``` - -`etw_audit` defaults to `false`. When enabled, the gateway account must be an -administrator or belong to the Windows Performance Log Users group. Workload -commands and working directories remain sandbox-scoped and must be supplied in -the `mxc` driver configuration when creating a sandbox. - -The driver records executable identity in process audit events and omits raw -command arguments because they can contain credentials or personal data. See -[OCSF JSON Export](/observability/ocsf-json-export) for durable Windows audit -output. - ### Docker -Sandboxes run as containers on a local bridge network. The supervisor binary is bind-mounted from the host (no in-cluster image pull required). Configure guest mTLS paths once under `[openshell.gateway]`; the gateway validates and injects the bundle into the selected local driver. +Sandboxes run as containers on a local bridge network. The supervisor binary is bind-mounted from the host (no in-cluster image pull required); guest mTLS material is supplied as host paths. ```toml [openshell] -version = 2 +version = 1 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" -compute_driver = "docker" -# Gateway-owned bundle injected into the selected local driver. -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" +compute_drivers = ["docker"] [openshell.drivers.docker] socket_path = "/var/run/docker.sock" @@ -698,13 +647,16 @@ grpc_endpoint = "https://host.openshell.internal:17670" # default to the gateway version; override either to pin a specific build. # sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:" # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" +guest_tls_ca = "/etc/openshell/certs/ca.pem" +guest_tls_cert = "/etc/openshell/certs/client.pem" +guest_tls_key = "/etc/openshell/certs/client-key.pem" network_name = "openshell-docker" host_gateway_ip = "172.17.0.1" # Unsafe operator override. Host bind mounts, including Docker local-driver # bind-backed volumes, expose gateway-host paths inside sandboxes and can # negate OpenShell isolation and filesystem controls. enable_bind_mounts = false -# Omit to use OpenShell's 2048-process default. Explicit 0 is invalid. +# Set to 0 to leave Docker's runtime default unchanged. sandbox_pids_limit = 2048 # Omit this field to keep Docker's runtime-selected AppArmor profile. # Localhost/ requires an operator-loaded profile; Unconfined is an @@ -722,25 +674,18 @@ proxy_auth_file = "/etc/openshell/secrets/proxy-auth" provider_spiffe_workload_api_socket = "/run/spire/agent.sock" ``` -Use `sandbox_label` for Docker configurations. The legacy -`sandbox_namespace` key is rejected. - ### Podman -Sandboxes run as Podman containers on a user-mode bridge network. The supervisor image is mounted read-only via Podman's `type=image` mount. Configure guest mTLS paths once under `[openshell.gateway]`; the gateway validates and injects the bundle into the selected local driver. +Each Podman sandbox has a workload container running `openshell-sandbox` with `network=none`, and a separate `openshell-supervisor` companion on the configured network. Both use non-root identities, drop all capabilities, and keep the runtime's default seccomp profile. A private named volume carries their authenticated gRPC Unix socket. Gateway JWTs, optional gateway mTLS material, and upstream proxy credentials are delivered only to the supervisor; user mounts and GPU devices stay with the workload. ```toml [openshell] -version = 2 +version = 1 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" -compute_driver = "podman" -# Gateway-owned bundle injected into the selected local driver. -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" +compute_drivers = ["podman"] [openshell.drivers.podman] # Rootless socket path. For root Podman use /run/podman/podman.sock. @@ -749,8 +694,7 @@ guest_tls_key = "/etc/openshell/certs/client-key.pem" # one. Set this to pin a specific Podman machine instead. socket_path = "/run/user/1000/podman/podman.sock" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "if_not_present" # always | if_not_present | never | newer -# Optional override. When omitted, the gateway derives this endpoint. +image_pull_policy = "missing" # always | missing | never | newer grpc_endpoint = "https://host.containers.internal:17670" # The gateway overwrites gateway_port from bind_address at runtime. gateway_port = 17670 @@ -758,19 +702,23 @@ network_name = "openshell" # Omit for the platform default: empty on Linux, 192.168.127.254 on macOS Podman machine. # Set "" to force Podman's host-gateway resolver. # host_gateway_ip = "192.168.127.254" -ssh_socket_path = "/run/openshell/ssh.sock" +sandbox_ssh_socket_path = "/run/openshell/ssh.sock" stop_timeout_secs = 45 # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" +guest_tls_ca = "/etc/openshell/certs/ca.pem" +guest_tls_cert = "/etc/openshell/certs/client.pem" +guest_tls_key = "/etc/openshell/certs/client-key.pem" # Unsafe operator override. Host bind mounts, including Podman local-driver # bind-backed volumes, expose gateway-host paths inside sandboxes and can # negate OpenShell isolation and filesystem controls. enable_bind_mounts = false -# Omit to use OpenShell's 2048-process default. Explicit 0 is invalid. +# Set to 0 to leave Podman's runtime default unchanged. sandbox_pids_limit = 2048 -# Health check interval in seconds. Omit to disable health checks; explicit 0 -# is invalid. Lower values detect readiness faster but increase process churn -# (each check spawns a conmon subprocess). +# Health check interval in seconds. Lower values detect readiness faster +# but increase process churn (each check spawns a conmon subprocess). +# Set to 0 to use a one-second check. Readiness checks cannot be disabled. +# Default: 10. health_check_interval_secs = 10 # User namespace mode for sandbox containers. Omit to use the default. # Supported modes: auto, host, keep-id, no-map, private. @@ -781,7 +729,7 @@ health_check_interval_secs = 10 # rootful Podman uses absolute host IDs (0:1000:1, 1:100000:65536). # uidmap = ["0:0:1", "1:1:65535"] # gidmap = ["0:0:1", "1:1:65535"] -# Corporate forward proxy for sandbox egress. When set, the in-container +# Corporate forward proxy for sandbox egress. When set, the external # supervisor chains policy-approved TLS tunnels through this proxy with HTTP # CONNECT instead of dialing destinations directly. Plain-HTTP requests are # not proxied and always dial the destination directly. http:// and https:// @@ -859,40 +807,21 @@ health_check_interval_secs = 10 # proxy_connect_by_hostname = true # Corporate CA trusted for an https:// proxy and TLS-intercepting proxies. # proxy_ca_bundle = "/etc/openshell/tls/proxy-ca.pem" -# Project a host Workload API Unix socket into the supervisor, or use an -# explicit container-reachable TCP endpoint, for provider token exchange. -# provider_spiffe_workload_api_socket = "/run/spire/agent.sock" -# provider_spiffe_workload_api_socket = "tcp:169.254.1.2:8081" -# Omit app_armor_profile to preserve Podman's runtime-selected profile. -# Set Unconfined only when the supervisor's mount setup requires it. -# Explicit RuntimeDefault and Localhost/ require Podman to report -# AppArmor support. -# app_armor_profile = "Unconfined" ``` -Use `ssh_socket_path` for Podman configurations. The legacy -`sandbox_ssh_socket_path` key is rejected. When `app_armor_profile` is omitted, -OpenShell sends no override and Podman applies its runtime-selected profile. -Set `Unconfined` explicitly only when the deployment requires the supervisor's -mount setup to bypass that profile. - ### MicroVM Each sandbox runs inside its own libkrun microVM managed by the standalone `openshell-driver-vm` subprocess. Use this driver when you want stronger isolation than container namespaces alone. ```toml [openshell] -version = 2 +version = 1 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" # VM is never auto-detected; an explicit entry here is required. -compute_driver = "vm" -# Gateway-owned bundle injected into the selected local driver. -guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" -guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" -guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" +compute_drivers = ["vm"] [openshell.drivers.vm] state_dir = "/var/lib/openshell/vm" @@ -908,20 +837,21 @@ krun_log_level = 1 vcpus = 2 mem_mib = 2048 overlay_disk_mib = 4096 -# Resolved sandbox UID/GID for new rootfs /etc/passwd entries. -# Defaults to the image's sandbox account, or 1000 when the account is absent; -# matching GID is used if sandbox_gid is empty. Persisted overlays recover their -# recorded identity, including 10001, rather than receiving a legacy fallback. -# Values must fall within OpenShell's allowed non-root sandbox identity range. +guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" +guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" +guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" +# Resolved sandbox UID/GID for the rootfs /etc/passwd entry. +# Defaults to 10001 when unset; matching GID is used if sandbox_gid is empty. +# Any non-root Linux UID/GID is valid. # sandbox_uid = 20001 -# sandbox_gid = 20001 # Corporate forward proxy for sandbox egress. The keys, their semantics, and # the fail-closed contract are identical to the Podman driver above: only TLS # (CONNECT) egress is chained, plain-HTTP destination requests always dial # directly, credentials must come from proxy_auth_file rather than the URL, # an http:// proxy with credentials requires proxy_auth_allow_insecure, and # any present-but-invalid value is rejected at gateway startup rather than -# degrading to a direct dial. proxy_auth_file is a path on the gateway host. +# degrading to a direct dial. proxy_auth_file and proxy_ca_bundle are paths on +# the gateway host. # # The sandbox cannot select or override these settings. The driver passes them # only to the host supervisor. @@ -937,18 +867,11 @@ overlay_disk_mib = 4096 # https_proxy = "http://host.openshell.internal:8080" # no_proxy = "10.0.0.0/8,.internal.example" # proxy_auth_file = "/etc/openshell/secrets/proxy-auth" -# An http:// proxy with proxy_auth_file requires this explicit acknowledgement: # proxy_auth_allow_insecure = true # Last resort for hostname-filtering proxy ACLs; see the Podman section above. # proxy_connect_by_hostname = true -# Gateway-host PEM bundle trusted for an https:// proxy and for server -# certificates re-signed by a TLS-intercepting proxy. Requires https_proxy. +# Corporate CA trusted for an https:// proxy and TLS-intercepting proxies. # proxy_ca_bundle = "/etc/openshell/tls/proxy-ca.pem" -# VM guests cannot mount a host Workload API Unix socket. Configure only a -# separately operated guest-reachable TCP listener and explicitly acknowledge -# the exposure; host-only sockets are never exposed automatically. -# provider_spiffe_workload_api_tcp_endpoint = "tcp:192.0.2.10:8081" -# provider_spiffe_allow_guest_tcp = true # Where the gateway stages rootfs tar archives for `--from ./rootfs.tar`. # Defaults to /rootfs-tar-staging. The gateway creates one # request-scoped subdirectory per staging slot and removes it after use. @@ -976,65 +899,13 @@ key used for driver-owned sandbox config such as `template.driver_config.` ```toml [openshell] -version = 2 +version = 1 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" -compute_driver = "kyma" +compute_drivers = ["kyma"] [openshell.drivers.kyma] socket_path = "/run/openshell/kyma-compute-driver.sock" ``` - -## Preflight package configuration {#gateway-config-preflight} - -Before starting a package-managed gateway, validate the selected file without -changing it: - -```shell -openshell-gateway config preflight --path ~/.config/openshell/gateway.toml -``` - -Without `--path`, the command validates a nonempty `OPENSHELL_GATEWAY_CONFIG`. -Otherwise, it validates an existing XDG gateway config when one is discovered. -When neither source selects a config, preflight succeeds. An explicit missing path, -a legacy schema-v1 file, invalid TOML, a symlink, or any nonregular file fails. -Preflight merges the selected file with the current `OPENSHELL_*` environment and -applies the daemon's read-only startup checks. These checks include selector and -socket normalization, registered-driver selection and configuration, rate-limit -pairs, TLS and mTLS relationships, interceptor registrations, and supervisor -middleware registrations. When a selected file omits `compute_driver`, preflight -validates each configured table for an auto-detectable driver without running the -runtime detection probes, which can connect local sockets or launch discovery -commands. It validates complete guest TLS path sets without requiring -package-generated certificates to exist before certificate generation. It does -not construct a compute driver or connect to a transport. A failed -preflight always preserves the file; it never migrates, replaces, or rewrites -configuration. - -To validate the exact daemon arguments that a wrapper will pass, place them after -`--` instead of using `--path`: - -```shell -openshell-gateway config preflight -- --config /etc/openshell/gateway.toml --grpc-rate-limit-requests 100 --grpc-rate-limit-window-seconds 60 -``` - -Debian and Ubuntu run preflight from the systemd user unit before local certificate -generation. The unit still loads the `gateway.env` environment file and starts the -gateway with no configuration arguments. Snap replays the exact effective daemon -arguments through preflight. It gives a nonempty `OPENSHELL_GATEWAY_CONFIG` -precedence; otherwise it validates and passes its canonical -`SNAP_COMMON/gateway.toml` only when that path exists in the filesystem. A broken -symlink is therefore rejected instead of being treated as absent. - -Package startup does not modify an operator-owned v1 file. Back it up, follow -[Migrate to schema version 2](#migrate-to-schema-version-2), then validate the -result explicitly before restarting the service: - -```shell -cp ~/.config/openshell/gateway.toml ~/.config/openshell/gateway.toml.v1.bak -$EDITOR ~/.config/openshell/gateway.toml -openshell-gateway config preflight --path ~/.config/openshell/gateway.toml -systemctl --user restart openshell-gateway -``` diff --git a/e2e/rust/tests/podman_gateway_start.rs b/e2e/rust/tests/podman_gateway_start.rs index 28a9ceb15c..7e159ffa78 100644 --- a/e2e/rust/tests/podman_gateway_start.rs +++ b/e2e/rust/tests/podman_gateway_start.rs @@ -38,10 +38,10 @@ const SANDBOX_NAME_LABEL: &str = "openshell.ai/sandbox-name"; /// harness), fall back to plain `podman`, leaving Linux behavior unchanged. fn podman_command() -> Command { let mut command = Command::new("podman"); - if let Ok(socket) = std::env::var("OPENSHELL_PODMAN_SOCKET") { - if !socket.is_empty() { - command.arg("--url").arg(format!("unix://{socket}")); - } + if let Ok(socket) = std::env::var("OPENSHELL_PODMAN_SOCKET") + && !socket.is_empty() + { + command.arg("--url").arg(format!("unix://{socket}")); } command } @@ -49,7 +49,15 @@ fn podman_command() -> Command { fn sandbox_container_running(sandbox_name: &str) -> Result { let sandbox_name_filter = format!("label={SANDBOX_NAME_LABEL}={sandbox_name}"); let output = podman_command() - .args(["ps", "-aq", "--filter", MANAGED_BY_LABEL_FILTER, "--filter"]) + .args([ + "ps", + "-aq", + "--filter", + MANAGED_BY_LABEL_FILTER, + "--filter", + "label=openshell.io/isolation-role=sandbox", + "--filter", + ]) .arg(sandbox_name_filter) .stdout(Stdio::piped()) .stderr(Stdio::piped()) diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs index e30516bf09..15ea13a77f 100644 --- a/e2e/rust/tests/podman_oci_identity.rs +++ b/e2e/rust/tests/podman_oci_identity.rs @@ -22,7 +22,7 @@ const BASE_IMAGE: &str = "ghcr.io/nvidia/openshell-community/sandboxes/base:late const READY_MARKER: &str = "podman-oci-identity-ready"; const OCI_UID: &str = "2345"; const OCI_GID: &str = "2346"; -const OCI_FALLBACK_POLICY: &str = r#"version: 1 +const OCI_FALLBACK_POLICY: &str = r"version: 1 filesystem_policy: include_workdir: true @@ -32,7 +32,7 @@ landlock: compatibility: best_effort network_policies: {} -"#; +"; struct ImageGuard { engine: ContainerEngine, @@ -128,7 +128,16 @@ fn run_engine(engine: &ContainerEngine, args: &[&str]) -> Result } fn sandbox_container_id(engine: &ContainerEngine, sandbox_name: &str) -> Result { + container_id_for_role(engine, sandbox_name, "sandbox") +} + +fn container_id_for_role( + engine: &ContainerEngine, + sandbox_name: &str, + role: &str, +) -> Result { let name_filter = format!("label=openshell.ai/sandbox-name={sandbox_name}"); + let role_filter = format!("label=openshell.io/isolation-role={role}"); let stdout = run_engine( engine, &[ @@ -138,6 +147,8 @@ fn sandbox_container_id(engine: &ContainerEngine, sandbox_name: &str) -> Result< "label=openshell.managed=true", "--filter", &name_filter, + "--filter", + &role_filter, ], )?; let ids = stdout @@ -233,5 +244,54 @@ async fn podman_uses_oci_identity_and_inspected_image_id() { "Podman sandbox must launch the immutable image ID inspected before creation" ); + assert_isolated_pair(&image, &sandbox, &container_id).await; sandbox.cleanup().await; } + +async fn assert_isolated_pair(image: &ImageGuard, sandbox: &SandboxGuard, container_id: &str) { + let supervisor_id = container_id_for_role(&image.engine, &sandbox.name, "supervisor") + .expect("find separate supervisor companion"); + assert_ne!(supervisor_id, container_id); + for id in [container_id, &supervisor_id] { + let user = run_engine( + &image.engine, + &["inspect", "--format", "{{.Config.User}}", id], + ) + .unwrap(); + assert_eq!(user, format!("{OCI_UID}:{OCI_GID}")); + let caps = run_engine( + &image.engine, + &["inspect", "--format", "{{.EffectiveCaps}}", id], + ) + .unwrap(); + assert_eq!( + caps, "[]", + "neither container may have effective capabilities" + ); + } + let network = run_engine( + &image.engine, + &[ + "inspect", + "--format", + "{{.HostConfig.NetworkMode}}", + container_id, + ], + ) + .unwrap(); + assert_eq!(network, "none"); + let mounts = run_engine( + &image.engine, + &[ + "inspect", + "--format", + "{{range .Mounts}}{{println .Destination}}{{end}}", + container_id, + ], + ) + .unwrap(); + assert!(!mounts.contains("/etc/openshell/tls")); + assert!(!mounts.contains("/.openshell/supervisor")); + let posture = sandbox.exec(&["sh", "-c", "set -eu; awk '/^CapEff:|^CapBnd:|^NoNewPrivs:/ {print}' /proc/self/status; test ! -r /.openshell/channel/sandbox/server.key; test ! -r /.openshell/supervisor/topology.payload"]).await.expect("workload cannot read either control credential set"); + assert!(posture.contains("0000000000000000")); +} diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index e7c47a251f..e20247cb6a 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -182,9 +182,24 @@ cleanup() { for id in ${sandbox_ids}; do local sandbox_id sandbox_id="$(podman_cmd inspect --format '{{ index .Config.Labels "openshell.ai/sandbox-id" }}' "${id}" 2>/dev/null || true)" - podman_cmd rm -f "${id}" >/dev/null 2>&1 || true if [ -n "${sandbox_id}" ] && [ "${sandbox_id}" != "" ]; then + # Only the companion is attached to the test network. Remove it first + # (it depends on the workload user namespace), then locate the isolated + # network=none workload by this test sandbox's immutable label. + podman_cmd rm -f "openshell-supervisor-${sandbox_id}" >/dev/null 2>&1 || true + local workload_ids workload_id + workload_ids="$(podman_cmd ps -aq --filter "label=openshell.managed=true" \ + --filter "label=openshell.ai/sandbox-id=${sandbox_id}" \ + --filter "label=openshell.io/isolation-role=sandbox" 2>/dev/null || true)" + for workload_id in ${workload_ids}; do + podman_cmd rm -f "${workload_id}" >/dev/null 2>&1 || true + done + podman_cmd volume rm "openshell-channel-${sandbox_id}" >/dev/null 2>&1 || true podman_cmd volume rm -f "openshell-sandbox-${sandbox_id}-workspace" >/dev/null 2>&1 || true + local secret_prefix + for secret_prefix in openshell-token openshell-proxy-auth openshell-tls-ca openshell-tls-cert openshell-tls-key; do + podman_cmd secret rm "${secret_prefix}-${sandbox_id}" >/dev/null 2>&1 || true + done fi done fi diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index a862db2f2f..5717b0e884 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -253,10 +253,14 @@ Common findings: - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Numeric workload identities `1` through `4294967294` are accepted; root, the invalid identity sentinel, and missing identities are rejected. - Supervisor cannot call back: check callback endpoint and gateway logs. -- A sandbox with explicit `protocol: tcp` endpoints fails before readiness: - inspect supervisor logs for policy DNS port-53 binding, synthetic-route, or - nftables redirect failures. Rootless Podman must provide these primitives - inside the supervisor-owned nested network namespace; setup fails closed. +- Inspect both Podman containers for the sandbox: the `sandbox` isolation role + must have network mode `none`; the `supervisor` role owns gateway callbacks + and egress. Both run non-root with all capabilities dropped. Check the private + channel volume and shared user-namespace mapping if authentication fails. +- If a sandbox fails before readiness, inspect its unprivileged enforcement + probe and the companion supervisor's private health check. Do not add + capabilities, attach a workload network, or disable the runtime seccomp + profile. There is no sandbox nftables or nested-network setup to repair. - Gateway exits before becoming healthy with a callback-listener discovery error: inspect `podman info --debug`, the configured Podman network, and the host's IPv4 default route. Rootless pasta uses the private source address From 64a9fa342de4a23321056f8a7120367ce674f42d Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 8 Sep 2026 17:23:28 -0700 Subject: [PATCH 02/19] fix(podman): stage bootstrap archives at named volume destinations Signed-off-by: Drew Newberry --- crates/openshell-driver-podman/README.md | 10 ++- crates/openshell-driver-podman/src/client.rs | 6 +- crates/openshell-driver-podman/src/driver.rs | 82 +++++++++++++++++-- .../openshell-driver-podman/src/isolation.rs | 63 +++++++------- .../openshell-driver-podman/src/test_utils.rs | 22 ++++- 5 files changed, 140 insertions(+), 43 deletions(-) diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 17bacce1ae..0768b9c712 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -30,6 +30,14 @@ The supervisor joins the workload's **user namespace only** to preserve UID/GID mapping for shared-volume access. PID, mount, and network namespaces remain separate. The channel volume uses shared SELinux relabeling (`:z`). +Before starting either container, the driver uploads volume-relative archives +directly to the channel and workspace volume destinations. A rootfs upload on a +stopped Podman container does not populate nested named volumes. Restart restores +only the channel bootstrap into the existing channel volume, preserving the +workspace. The workload starts before the supervisor so its user namespace exists +when the supervisor joins it; a stopped supervisor resolves that namespace again +on its next start. + The runtime must pass the sandbox's unprivileged enforcement probe, including nested seccomp notification and Landlock. Unsupported runtime defaults fail closed; do not switch to an unconfined profile or add capabilities. @@ -77,7 +85,7 @@ children, never the supervisor process. ## Lifecycle and readiness -Create builds both stopped containers and stages both private archives before +Create builds both stopped containers and stages the private archives before starting either container. The sandbox does not execute the agent until the supervisor authenticates and confirms the common boundary contract. Failed creation removes only containers created by that attempt, then cleans up diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 7ab13d0a42..be5dcea0b9 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -483,13 +483,17 @@ impl PodmanClient { pub(crate) async fn copy_to_container( &self, name: &str, + destination: &str, archive: Vec, ) -> Result<(), PodmanApiError> { validate_name(name)?; let (status, bytes) = self .request_raw( hyper::Method::PUT, - &format!("/libpod/containers/{name}/archive?path=/"), + &format!( + "/libpod/containers/{name}/archive?path={}", + url_encode(destination) + ), "application/x-tar", archive.into(), ) diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index f7d20cb374..203786504a 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -1016,7 +1016,14 @@ impl PodmanComputeDriver { child_env, )?; self.client - .copy_to_container(&workload_id, archives.workload) + .copy_to_container( + &workload_id, + crate::isolation::CHANNEL_ROOT, + archives.channel, + ) + .await?; + self.client + .copy_to_container(&workload_id, "/sandbox", archives.workspace) .await?; specs.supervisor.join_user_namespace(&workload_id); let supervisor_id = self @@ -1025,7 +1032,7 @@ impl PodmanComputeDriver { .await?; created_supervisor = Some(supervisor_id.clone()); self.client - .copy_to_container(&supervisor_id, archives.supervisor) + .copy_to_container(&supervisor_id, "/", archives.supervisor) .await?; // Both resources and private files exist before either // container can run. Only the trusted sandbox starts here; @@ -1312,7 +1319,9 @@ impl PodmanComputeDriver { .await?; let bundle = extract_first_tar_entry(&archive).map_err(ComputeDriverError::Precondition)?; - self.client.copy_to_container(&container_id, bundle).await?; + self.client + .copy_to_container(&container_id, crate::isolation::CHANNEL_ROOT, bundle) + .await?; self.client.verify_isolation_fence(&container_id).await?; self.client.start_container(&container_id).await?; if let Err(error) = self.client.start_container(&supervisor).await { @@ -1906,6 +1915,24 @@ mod tests { .await .expect("start should succeed"); start_handle.await.expect("start stub should finish"); + let restart_requests = start_requests.lock().unwrap().clone(); + assert_eq!( + restart_requests + .iter() + .filter(|request| request.starts_with("PUT ")) + .cloned() + .collect::>(), + vec![format!( + "PUT {}", + api_path("/libpod/containers/ctr-1/archive?path=%2F.openshell%2Fchannel") + )] + ); + assert!( + !restart_requests + .iter() + .any(|request| request.contains("/volumes/create") + || request.contains("/containers/create")) + ); assert_eq!( start_requests .lock() @@ -2054,7 +2081,7 @@ mod tests { use tracing_subscriber::layer::SubscriberExt as _; let _tracing_lock = openshell_otel_test_support::tracing_test_lock().await; - let (socket_path, _requests, handle) = spawn_podman_stub( + let (socket_path, requests, handle) = spawn_podman_stub( "trace-create", create_setup_responses(false) .into_iter() @@ -2074,6 +2101,22 @@ mod tests { .await .expect("create should succeed"); handle.await.expect("stub should finish"); + let uploads: Vec<_> = requests + .lock() + .unwrap() + .iter() + .filter(|request| request.starts_with("PUT ")) + .cloned() + .collect(); + assert_eq!( + uploads, + [ + "/libpod/containers/workload/archive?path=%2F.openshell%2Fchannel", + "/libpod/containers/workload/archive?path=%2Fsandbox", + "/libpod/containers/supervisor/archive?path=%2F", + ] + .map(|path| format!("PUT {}", api_path(path))) + ); provider.force_flush().unwrap(); let spans = exporter.get_finished_spans().unwrap(); @@ -3121,7 +3164,18 @@ mod tests { fn restart_responses() -> Vec { let mut archive = tar::Builder::new(Vec::new()); - let bundle = tar::Builder::new(Vec::new()).into_inner().unwrap(); + let identity = openshell_isolation_interface::contract::ResolvedWorkloadIdentity::new( + 1000, + 1001, + vec![], + "image".into(), + "sha256:image".into(), + ) + .unwrap(); + let bundle = + crate::isolation::bootstrap_archives("sandbox-1", "ctr-1", &identity, HashMap::new()) + .unwrap() + .channel; let mut header = tar::Header::new_gnu(); header.set_size(bundle.len() as u64); header.set_mode(0o600); @@ -3136,7 +3190,7 @@ mod tests { r#"{"Id":"supervisor","Name":"supervisor","State":{"Status":"exited","Running":false},"Config":{}}"#, ), StubResponse::new(StatusCode::OK, archive.into_inner().unwrap()), - StubResponse::new(StatusCode::OK, ""), // restore bootstrap + StubResponse::new(StatusCode::OK, "").with_archive_members(channel_archive_members()), fence_response(), StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor start @@ -3234,7 +3288,8 @@ mod tests { vec![ created_response("workload"), fence_response(), - StubResponse::new(StatusCode::OK, ""), // workload archive + StubResponse::new(StatusCode::OK, "").with_archive_members(channel_archive_members()), + StubResponse::new(StatusCode::OK, "").with_archive_members(&["."]), created_response("supervisor"), StubResponse::new(StatusCode::OK, ""), // supervisor archive StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start @@ -3242,6 +3297,17 @@ mod tests { ] } + fn channel_archive_members() -> &'static [&'static str] { + &[ + ".", + "sandbox", + "sandbox/bootstrap.json", + "sandbox/server.crt", + "sandbox/server.key", + "sandbox/client-ca.crt", + ] + } + #[tokio::test] async fn reserved_image_control_root_fails_before_workload_or_secrets() { let (path, requests, handle) = spawn_podman_stub( @@ -3320,7 +3386,7 @@ mod tests { "create-start-fail", create_setup_responses(true) .into_iter() - .chain(create_launch_responses().into_iter().take(6)) + .chain(create_launch_responses().into_iter().take(7)) .chain([ StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, "supervisor start failed"), StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor diff --git a/crates/openshell-driver-podman/src/isolation.rs b/crates/openshell-driver-podman/src/isolation.rs index 9a33e6559f..1ba6aae055 100644 --- a/crates/openshell-driver-podman/src/isolation.rs +++ b/crates/openshell-driver-podman/src/isolation.rs @@ -122,7 +122,8 @@ pub fn resolve_identity( } pub struct BootstrapArchives { - pub workload: Vec, + pub channel: Vec, + pub workspace: Vec, pub supervisor: Vec, } @@ -194,28 +195,22 @@ pub fn bootstrap_archives( workload_identity: identity.clone(), driver_fence, }; - let mut workload = Archive::new(identity); - workload.directory(".openshell", 0o755, false)?; - workload.directory(".openshell/channel", 0o755, false)?; - workload.directory(".openshell/channel/sandbox", 0o711, true)?; - workload.directory("sandbox", 0o700, true)?; - workload.file( - BOOTSTRAP_PATH, + // Libpod resolves the requested upload destination once for a stopped + // container. Archive entries must be relative to the selected named volume, + // not rootfs paths that the volume would shadow on container start. + let mut channel = Archive::new(identity); + channel.directory(".", 0o755, false)?; + channel.directory("sandbox", 0o711, true)?; + channel.file( + "sandbox/bootstrap.json", &serde_json::to_vec(&config).map_err(invalid)?, )?; - workload.file( - "/.openshell/channel/sandbox/server.crt", - tls.sandbox_certificate_pem.as_bytes(), - )?; - workload.file( - "/.openshell/channel/sandbox/server.key", - tls.sandbox_private_key_pem.as_bytes(), - )?; - workload.file( - "/.openshell/channel/sandbox/client-ca.crt", - tls.ca_certificate_pem.as_bytes(), - )?; - let workload = workload.finish()?; + channel.file("sandbox/server.crt", tls.sandbox_certificate_pem.as_bytes())?; + channel.file("sandbox/server.key", tls.sandbox_private_key_pem.as_bytes())?; + channel.file("sandbox/client-ca.crt", tls.ca_certificate_pem.as_bytes())?; + let channel = channel.finish()?; + let mut workspace = Archive::new(identity); + workspace.directory(".", 0o700, true)?; let mut supervisor = Archive::new(identity); supervisor.directory(".openshell", 0o755, false)?; supervisor.directory(".openshell/supervisor", 0o700, true)?; @@ -223,9 +218,10 @@ pub fn bootstrap_archives( TOPOLOGY_PATH, &serde_json::to_vec(&topology).map_err(invalid)?, )?; - supervisor.file(RESTART_BUNDLE_PATH, &workload)?; + supervisor.file(RESTART_BUNDLE_PATH, &channel)?; Ok(BootstrapArchives { - workload, + channel, + workspace: workspace.finish()?, supervisor: supervisor.finish()?, }) } @@ -337,15 +333,20 @@ mod tests { .unwrap(); let archives = bootstrap_archives("sandbox", "container", &identity, HashMap::new()).unwrap(); - let workload = files(&archives.workload); + let workload = files(&archives.channel); let supervisor = files(&archives.supervisor); + let mut workspace = tar::Archive::new(archives.workspace.as_slice()); + let mut entries = workspace.entries().unwrap(); + let root = entries.next().unwrap().unwrap(); + assert_eq!(root.path().unwrap().as_ref(), std::path::Path::new(".")); + assert!(root.header().entry_type().is_dir()); + assert_eq!(root.header().uid().unwrap(), u64::from(identity.uid)); + assert_eq!(root.header().gid().unwrap(), u64::from(identity.gid)); + assert_eq!(root.header().mode().unwrap(), 0o700); + assert!(entries.next().is_none()); assert_eq!(workload.len(), 4); assert_eq!(supervisor.len(), 2); - assert!( - workload - .keys() - .all(|path| path.starts_with(".openshell/channel/sandbox")) - ); + assert!(workload.keys().all(|path| path.starts_with("sandbox"))); assert!( supervisor .keys() @@ -353,7 +354,7 @@ mod tests { ); let config: BoundaryConfig = serde_json::from_slice( workload - .get(&PathBuf::from(BOOTSTRAP_PATH.trim_start_matches('/'))) + .get(&PathBuf::from("sandbox/bootstrap.json")) .unwrap(), ) .unwrap(); @@ -375,7 +376,7 @@ mod tests { supervisor .get(&PathBuf::from(RESTART_BUNDLE_PATH.trim_start_matches('/'))) .unwrap(), - &archives.workload + &archives.channel ); } } diff --git a/crates/openshell-driver-podman/src/test_utils.rs b/crates/openshell-driver-podman/src/test_utils.rs index 24e9d4d8ed..25cbcb4ac2 100644 --- a/crates/openshell-driver-podman/src/test_utils.rs +++ b/crates/openshell-driver-podman/src/test_utils.rs @@ -3,7 +3,7 @@ //! Shared test helpers for openshell-driver-podman unit tests. -use http_body_util::Full; +use http_body_util::{BodyExt as _, Full}; use hyper::StatusCode; use hyper::body::Bytes; use hyper::server::conn::http1; @@ -22,6 +22,7 @@ pub struct StubResponse { pub status: StatusCode, pub body: Bytes, pub delay: Duration, + pub archive_members: Option>, } impl StubResponse { @@ -30,6 +31,7 @@ impl StubResponse { status, body: body.into(), delay: Duration::ZERO, + archive_members: None, } } @@ -37,6 +39,11 @@ impl StubResponse { self.delay = delay; self } + + pub fn with_archive_members(mut self, members: &[&str]) -> Self { + self.archive_members = Some(members.iter().map(PathBuf::from).collect()); + self + } } /// Generate a unique Unix socket path for a test. @@ -88,7 +95,7 @@ pub fn spawn_podman_stub( let result = http1::Builder::new() .serve_connection( TokioIo::new(stream), - service_fn(move |req| { + service_fn(move |req: hyper::Request| { let log = log.clone(); let queue = queue.clone(); async move { @@ -104,6 +111,17 @@ pub fn spawn_podman_stub( .expect("response queue lock should not be poisoned") .pop_front() .expect("stub response should exist"); + if let Some(expected_members) = &response.archive_members { + assert_eq!(req.method(), hyper::Method::PUT); + let body = req.into_body().collect().await.unwrap().to_bytes(); + let mut archive = tar::Archive::new(body.as_ref()); + let members: Vec<_> = archive + .entries() + .unwrap() + .map(|entry| entry.unwrap().path().unwrap().into_owned()) + .collect(); + assert_eq!(&members, expected_members); + } tokio::time::sleep(response.delay).await; Ok::<_, Infallible>( hyper::Response::builder() From b2bffa70efedc2c02f96e0cd8897ddfb8938a107 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 10 Sep 2026 18:59:24 -0700 Subject: [PATCH 03/19] feat(podman): rotate launch-scoped authentication Signed-off-by: Drew Newberry --- .../openshell-driver-podman/src/container.rs | 1 + crates/openshell-driver-podman/src/driver.rs | 117 +++++++++++++++-- crates/openshell-driver-podman/src/grpc.rs | 2 +- .../openshell-driver-podman/src/isolation.rs | 122 +++++++++++++----- 4 files changed, 195 insertions(+), 47 deletions(-) diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 7091fbb7a2..839a9f1bd3 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1466,6 +1466,7 @@ pub fn build_isolation_specs( "--topology-payload-file={}", crate::isolation::TOPOLOGY_PATH ), + format!("--auth-bundle-file={}", crate::isolation::AUTH_BUNDLE_PATH), "--health-socket-path=/run/openshell/supervisor-health.sock".into(), ]); supervisor.env.insert( diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 203786504a..9270d6bf35 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -41,6 +41,24 @@ use url::Url; const STOP_COMPLETION_POLL_INTERVAL: Duration = Duration::from_millis(50); const STOP_COMPLETION_TIMEOUT_HEADROOM: Duration = Duration::from_secs(5); +fn decode_launch_authentication( + encoded: &[u8], +) -> Result { + let authentication = + serde_json::from_slice::(encoded) + .map_err(|error| { + ComputeDriverError::Precondition(format!( + "decode Podman sandbox launch authentication: {error}" + )) + })?; + authentication.validate().map_err(|error| { + ComputeDriverError::Precondition(format!( + "validate Podman sandbox launch authentication: {error}" + )) + })?; + Ok(authentication) +} + impl From for ComputeDriverError { fn from(value: PodmanApiError) -> Self { match value { @@ -1009,11 +1027,24 @@ impl PodmanComputeDriver { .map(|(key, value)| (key.into(), value.into())) }) .collect(); + let launch_authentication = sandbox + .spec + .as_ref() + .filter(|spec| !spec.launch_authentication.is_empty()) + .ok_or_else(|| { + ComputeDriverError::Precondition( + "Podman sandbox launch authentication is required".to_string(), + ) + }) + .and_then(|spec| { + decode_launch_authentication(&spec.launch_authentication) + })?; let archives = crate::isolation::bootstrap_archives( &sandbox.id, &workload_id, &identity, child_env, + &launch_authentication, )?; self.client .copy_to_container( @@ -1270,8 +1301,13 @@ impl PodmanComputeDriver { sandbox.id = %sandbox_id, ) )] - pub async fn start_sandbox(&self, sandbox_id: &str) -> Result<(), ComputeDriverError> { + pub async fn start_sandbox( + &self, + sandbox_id: &str, + encoded_authentication: &[u8], + ) -> Result<(), ComputeDriverError> { let span_status = openshell_otel::ErrorStatusGuard::current(); + let launch_authentication = decode_launch_authentication(encoded_authentication)?; let container = self .find_container(sandbox_id) .await? @@ -1319,8 +1355,23 @@ impl PodmanComputeDriver { .await?; let bundle = extract_first_tar_entry(&archive).map_err(ComputeDriverError::Precondition)?; + let previous_config = crate::isolation::boundary_config_from_channel_archive(&bundle)?; + let archives = crate::isolation::bootstrap_archives( + sandbox_id, + &container_id, + &previous_config.workload_identity, + previous_config.child_env, + &launch_authentication, + )?; + self.client + .copy_to_container( + &container_id, + crate::isolation::CHANNEL_ROOT, + archives.channel, + ) + .await?; self.client - .copy_to_container(&container_id, crate::isolation::CHANNEL_ROOT, bundle) + .copy_to_container(&supervisor, "/", archives.supervisor) .await?; self.client.verify_isolation_fence(&container_id).await?; self.client.start_container(&container_id).await?; @@ -1787,6 +1838,10 @@ mod tests { use super::*; use crate::test_utils::{StubResponse, spawn_podman_stub}; use hyper::StatusCode; + use openshell_core::jwt::{ + CredentialEpoch, SandboxLaunchAuthentication, SecretJwt, SessionVerificationKey, + SupervisorAuthBundle, + }; use openshell_core::proto::compute::v1::{ DriverSandboxSpec, DriverSandboxTemplate, ResourceRequirements, }; @@ -1794,6 +1849,28 @@ mod tests { use std::fs; use std::path::{Path, PathBuf}; + fn launch_authentication() -> SandboxLaunchAuthentication { + SandboxLaunchAuthentication { + supervisor: SupervisorAuthBundle { + session_id: openshell_core::SandboxSessionId::new(), + gateway_token: SecretJwt::parse("gateway.token.value").unwrap(), + gateway_expires_at: i64::MAX, + sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(), + sandbox_expires_at: i64::MAX, + credential_epoch: CredentialEpoch::new(1).unwrap(), + }, + gateway_id: "gateway-test".to_string(), + verification_keys: vec![SessionVerificationKey { + key_id: "test-key".to_string(), + public_key_pem: b"public-key".to_vec(), + }], + } + } + + fn encoded_launch_authentication() -> Vec { + serde_json::to_vec(&launch_authentication()).unwrap() + } + // ── socket resolution ─────────────────────────────────────────────── // // These test resolve_socket_path directly with an injected detector, so @@ -1910,8 +1987,9 @@ mod tests { ), ].into_iter().chain(restart_responses()).collect(), ); + let authentication = encoded_launch_authentication(); test_driver(start_socket.clone()) - .start_sandbox("sandbox-1") + .start_sandbox("sandbox-1", &authentication) .await .expect("start should succeed"); start_handle.await.expect("start stub should finish"); @@ -1922,10 +2000,16 @@ mod tests { .filter(|request| request.starts_with("PUT ")) .cloned() .collect::>(), - vec![format!( - "PUT {}", - api_path("/libpod/containers/ctr-1/archive?path=%2F.openshell%2Fchannel") - )] + vec![ + format!( + "PUT {}", + api_path("/libpod/containers/ctr-1/archive?path=%2F.openshell%2Fchannel") + ), + format!( + "PUT {}", + api_path("/libpod/containers/openshell-supervisor-sandbox-1/archive?path=%2F") + ), + ] ); assert!( !restart_requests @@ -2209,8 +2293,9 @@ mod tests { ), ].into_iter().chain(restart_responses()).collect(), ); + let authentication = encoded_launch_authentication(); test_driver(start_socket.clone()) - .start_sandbox("sandbox-1") + .start_sandbox("sandbox-1", &authentication) .with_subscriber(subscriber) .await .expect("start should succeed"); @@ -3172,10 +3257,16 @@ mod tests { "sha256:image".into(), ) .unwrap(); - let bundle = - crate::isolation::bootstrap_archives("sandbox-1", "ctr-1", &identity, HashMap::new()) - .unwrap() - .channel; + let authentication = launch_authentication(); + let bundle = crate::isolation::bootstrap_archives( + "sandbox-1", + "ctr-1", + &identity, + HashMap::new(), + &authentication, + ) + .unwrap() + .channel; let mut header = tar::Header::new_gnu(); header.set_size(bundle.len() as u64); header.set_mode(0o600); @@ -3191,6 +3282,7 @@ mod tests { ), StubResponse::new(StatusCode::OK, archive.into_inner().unwrap()), StubResponse::new(StatusCode::OK, "").with_archive_members(channel_archive_members()), + StubResponse::new(StatusCode::OK, ""), // refreshed supervisor auth and topology fence_response(), StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor start @@ -3304,7 +3396,6 @@ mod tests { "sandbox/bootstrap.json", "sandbox/server.crt", "sandbox/server.key", - "sandbox/client-ca.crt", ] } diff --git a/crates/openshell-driver-podman/src/grpc.rs b/crates/openshell-driver-podman/src/grpc.rs index d18ad5d88d..94d2b31a10 100644 --- a/crates/openshell-driver-podman/src/grpc.rs +++ b/crates/openshell-driver-podman/src/grpc.rs @@ -200,7 +200,7 @@ impl ComputeDriver for ComputeDriverService { return Err(Status::invalid_argument("sandbox_id is required")); } self.driver - .start_sandbox(&request.sandbox_id) + .start_sandbox(&request.sandbox_id, &request.launch_authentication) .await .map_err(Status::from)?; Ok(Response::new(StartSandboxResponse {})) diff --git a/crates/openshell-driver-podman/src/isolation.rs b/crates/openshell-driver-podman/src/isolation.rs index 1ba6aae055..6eb475149b 100644 --- a/crates/openshell-driver-podman/src/isolation.rs +++ b/crates/openshell-driver-podman/src/isolation.rs @@ -4,13 +4,15 @@ //! Podman-owned provisioning for the common authenticated isolation channel. use std::collections::{BTreeMap, HashMap}; +use std::io::Read; use std::path::PathBuf; use openshell_core::ComputeDriverError; use openshell_core::proto::compute::v1::DriverSandbox; use openshell_isolation_interface::boundary_protocol::{ - BoundaryClientTls, BoundaryConfig, BoundaryListener, BoundaryServerTls, BoundaryTopology, - BoundaryTransport, generate_boundary_mutual_tls_material, + BoundaryConfig, BoundaryListener, BoundaryTopology, GatewayVerificationKey, + SandboxTlsClientConfig, SandboxTlsServerConfig, SandboxTransport, + generate_sandbox_tls_material, }; use openshell_isolation_interface::contract::{DriverFenceEvidence, ResolvedWorkloadIdentity}; @@ -19,6 +21,7 @@ pub const WORKLOAD_FILTER: &str = "openshell.io/isolation-role=sandbox"; pub const CHANNEL_ROOT: &str = "/.openshell/channel"; pub const BOOTSTRAP_PATH: &str = "/.openshell/channel/sandbox/bootstrap.json"; pub const TOPOLOGY_PATH: &str = "/.openshell/supervisor/topology.payload"; +pub const AUTH_BUNDLE_PATH: &str = "/.openshell/supervisor/auth.json"; pub const RESTART_BUNDLE_PATH: &str = "/.openshell/supervisor/sandbox-bundle.tar"; const SOCKET_PATH: &str = "/.openshell/channel/sandbox/control.sock"; @@ -134,8 +137,11 @@ pub fn bootstrap_archives( container_id: &str, identity: &ResolvedWorkloadIdentity, child_env: HashMap, + launch_authentication: &openshell_core::jwt::SandboxLaunchAuthentication, ) -> Result { - let tls = generate_boundary_mutual_tls_material().map_err(invalid)?; + launch_authentication.validate().map_err(invalid)?; + let session_id = launch_authentication.supervisor.session_id; + let tls = generate_sandbox_tls_material(session_id).map_err(invalid)?; let resource_claims = BTreeMap::from([ ("podman.container_id".into(), container_id.into()), ( @@ -149,25 +155,29 @@ pub fn bootstrap_archives( unexpected_networks: Vec::new(), }; let generation = uuid::Uuid::new_v4().to_string(); - let session_epoch = uuid::Uuid::new_v4().to_string(); - let bootstrap_token = format!( - "{}{}", - uuid::Uuid::new_v4().simple(), - uuid::Uuid::new_v4().simple() - ); + let verification_keys = launch_authentication + .verification_keys + .iter() + .map(|key| { + String::from_utf8(key.public_key_pem.clone()) + .map(|public_key_pem| GatewayVerificationKey { + key_id: key.key_id.clone(), + public_key_pem, + }) + .map_err(invalid) + }) + .collect::, _>>()?; let config = BoundaryConfig { boundary_id: sandbox_id.into(), generation: generation.clone(), - session_epoch: session_epoch.clone(), - bootstrap_token: bootstrap_token.clone(), + session_id, + gateway_id: launch_authentication.gateway_id.clone(), + verification_keys, listener: BoundaryListener::Unix { socket_path: PathBuf::from(SOCKET_PATH), - tls: BoundaryServerTls { + tls: SandboxTlsServerConfig { certificate_chain_path: PathBuf::from("/.openshell/channel/sandbox/server.crt"), private_key_path: PathBuf::from("/.openshell/channel/sandbox/server.key"), - client_ca_certificate_path: PathBuf::from( - "/.openshell/channel/sandbox/client-ca.crt", - ), }, }, resource_claims: resource_claims.clone(), @@ -179,16 +189,13 @@ pub fn bootstrap_archives( let topology = BoundaryTopology { boundary_id: sandbox_id.into(), generation, - session_epoch, - bootstrap_token, - transport: BoundaryTransport::Unix { + session_id, + transport: SandboxTransport::Unix { socket_path: PathBuf::from(SOCKET_PATH), - tls: BoundaryClientTls { - server_name: tls.server_name, - ca_certificate_pem: tls.ca_certificate_pem.clone(), - certificate_chain_pem: tls.supervisor_certificate_pem, - private_key_pem: tls.supervisor_private_key_pem, - }, + }, + tls: SandboxTlsClientConfig { + server_name: tls.server_name, + trust_anchor_pem: tls.trust_anchor_pem, }, host_gateway_ip: None, resource_claims, @@ -205,9 +212,8 @@ pub fn bootstrap_archives( "sandbox/bootstrap.json", &serde_json::to_vec(&config).map_err(invalid)?, )?; - channel.file("sandbox/server.crt", tls.sandbox_certificate_pem.as_bytes())?; - channel.file("sandbox/server.key", tls.sandbox_private_key_pem.as_bytes())?; - channel.file("sandbox/client-ca.crt", tls.ca_certificate_pem.as_bytes())?; + channel.file("sandbox/server.crt", tls.certificate_chain_pem.as_bytes())?; + channel.file("sandbox/server.key", tls.private_key_pem.as_bytes())?; let channel = channel.finish()?; let mut workspace = Archive::new(identity); workspace.directory(".", 0o700, true)?; @@ -218,6 +224,10 @@ pub fn bootstrap_archives( TOPOLOGY_PATH, &serde_json::to_vec(&topology).map_err(invalid)?, )?; + supervisor.file( + AUTH_BUNDLE_PATH, + &serde_json::to_vec(&launch_authentication.supervisor).map_err(invalid)?, + )?; supervisor.file(RESTART_BUNDLE_PATH, &channel)?; Ok(BootstrapArchives { channel, @@ -226,6 +236,24 @@ pub fn bootstrap_archives( }) } +pub fn boundary_config_from_channel_archive( + archive: &[u8], +) -> Result { + for entry in tar::Archive::new(archive).entries().map_err(invalid)? { + let mut entry = entry.map_err(invalid)?; + if entry.path().map_err(invalid)?.as_ref() != std::path::Path::new("sandbox/bootstrap.json") + { + continue; + } + let mut bytes = Vec::new(); + entry.read_to_end(&mut bytes).map_err(invalid)?; + return serde_json::from_slice(&bytes).map_err(invalid); + } + Err(ComputeDriverError::Precondition( + "Podman restart bundle has no sandbox bootstrap".to_string(), + )) +} + struct Archive<'a> { builder: tar::Builder>, identity: &'a ResolvedWorkloadIdentity, @@ -285,7 +313,28 @@ impl<'a> Archive<'a> { #[cfg(test)] mod tests { use super::*; - use std::io::Read as _; + use openshell_core::jwt::{ + CredentialEpoch, SandboxLaunchAuthentication, SecretJwt, SessionVerificationKey, + SupervisorAuthBundle, + }; + + fn authentication() -> SandboxLaunchAuthentication { + SandboxLaunchAuthentication { + supervisor: SupervisorAuthBundle { + session_id: openshell_core::SandboxSessionId::new(), + gateway_token: SecretJwt::parse("gateway.token.value").unwrap(), + gateway_expires_at: i64::MAX, + sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(), + sandbox_expires_at: i64::MAX, + credential_epoch: CredentialEpoch::new(1).unwrap(), + }, + gateway_id: "gateway-test".to_string(), + verification_keys: vec![SessionVerificationKey { + key_id: "test-key".to_string(), + public_key_pem: b"public-key".to_vec(), + }], + } + } #[test] fn identity_uses_pinned_image_accounts_and_rejects_root() { @@ -331,8 +380,15 @@ mod tests { "sha256:image".into(), ) .unwrap(); - let archives = - bootstrap_archives("sandbox", "container", &identity, HashMap::new()).unwrap(); + let authentication = authentication(); + let archives = bootstrap_archives( + "sandbox", + "container", + &identity, + HashMap::new(), + &authentication, + ) + .unwrap(); let workload = files(&archives.channel); let supervisor = files(&archives.supervisor); let mut workspace = tar::Archive::new(archives.workspace.as_slice()); @@ -344,8 +400,8 @@ mod tests { assert_eq!(root.header().gid().unwrap(), u64::from(identity.gid)); assert_eq!(root.header().mode().unwrap(), 0o700); assert!(entries.next().is_none()); - assert_eq!(workload.len(), 4); - assert_eq!(supervisor.len(), 2); + assert_eq!(workload.len(), 3); + assert_eq!(supervisor.len(), 3); assert!(workload.keys().all(|path| path.starts_with("sandbox"))); assert!( supervisor @@ -365,7 +421,7 @@ mod tests { ) .unwrap(); assert_eq!(config.boundary_id, topology.boundary_id); - assert_eq!(config.bootstrap_token, topology.bootstrap_token); + assert_eq!(config.session_id, topology.session_id); assert_eq!(config.driver_fence, topology.driver_fence); assert_eq!(config.workload_identity, identity); topology From d91df8786075f446a4da4c7bfe33b980463c1d8e Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 11 Sep 2026 10:05:44 -0700 Subject: [PATCH 04/19] refactor(podman): use sandbox backend protocol Signed-off-by: Drew Newberry --- Cargo.lock | 1 + crates/openshell-driver-podman/Cargo.toml | 1 + crates/openshell-driver-podman/src/isolation.rs | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 27b831823c..2e1d017d18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4223,6 +4223,7 @@ dependencies = [ "openshell-isolation-interface", "openshell-otel", "openshell-otel-test-support", + "openshell-sandbox-backend", "opentelemetry", "opentelemetry_sdk", "prost-types", diff --git a/crates/openshell-driver-podman/Cargo.toml b/crates/openshell-driver-podman/Cargo.toml index cd463b86fe..c49c309f65 100644 --- a/crates/openshell-driver-podman/Cargo.toml +++ b/crates/openshell-driver-podman/Cargo.toml @@ -18,6 +18,7 @@ path = "src/main.rs" openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } openshell-otel = { path = "../openshell-otel" } openshell-isolation-interface = { path = "../openshell-isolation-interface" } +openshell-sandbox-backend = { path = "../openshell-sandbox-backend" } tar = "0.4" uuid = { workspace = true } diff --git a/crates/openshell-driver-podman/src/isolation.rs b/crates/openshell-driver-podman/src/isolation.rs index 6eb475149b..8050ae4790 100644 --- a/crates/openshell-driver-podman/src/isolation.rs +++ b/crates/openshell-driver-podman/src/isolation.rs @@ -9,12 +9,12 @@ use std::path::PathBuf; use openshell_core::ComputeDriverError; use openshell_core::proto::compute::v1::DriverSandbox; -use openshell_isolation_interface::boundary_protocol::{ +use openshell_isolation_interface::contract::{DriverFenceEvidence, ResolvedWorkloadIdentity}; +use openshell_sandbox_backend::boundary_protocol::{ BoundaryConfig, BoundaryListener, BoundaryTopology, GatewayVerificationKey, SandboxTlsClientConfig, SandboxTlsServerConfig, SandboxTransport, generate_sandbox_tls_material, }; -use openshell_isolation_interface::contract::{DriverFenceEvidence, ResolvedWorkloadIdentity}; pub const LABEL_ROLE: &str = "openshell.io/isolation-role"; pub const WORKLOAD_FILTER: &str = "openshell.io/isolation-role=sandbox"; From 294bb50535efd21f1fff67b446a881c65a207f7f Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 11 Sep 2026 10:46:02 -0700 Subject: [PATCH 05/19] refactor(podman): use host networking for supervisor Signed-off-by: Drew Newberry --- crates/openshell-driver-podman/NETWORKING.md | 11 ++++++----- crates/openshell-driver-podman/README.md | 4 ++-- crates/openshell-driver-podman/src/container.rs | 8 ++++++++ docs/reference/sandbox-compute-drivers.mdx | 2 ++ 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index 9a0f96475f..4ad9eedae6 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -7,7 +7,7 @@ environment variable. ```text workload container supervisor container -agent -> sandbox -- private UDS / gRPC -> policy proxy -> Podman network -> destination +agent -> sandbox -- private UDS / gRPC -> policy proxy -> host network -> destination | +-- authenticated gateway callback ``` @@ -26,9 +26,10 @@ supervisor. General UDP is unsupported. ## Supervisor callback network -The configured `network_name`, host-gateway aliases, upstream corporate proxy, -and published SSH port apply only to the supervisor companion. The gateway's -SSH tunnel still uses the supervisor relay, not the published port. +The supervisor companion uses Podman's host network. Host-gateway aliases and +the upstream corporate proxy apply only to the supervisor. The gateway's SSH +tunnel uses the supervisor relay over its private Unix socket, so the driver +does not publish a supervisor port. Rootful Podman uses the configured bridge and its gateway address. Rootless local callbacks require the existing pasta path; slirp4netns or unknown helpers @@ -50,7 +51,7 @@ Inspect both containers with the same sandbox-ID label, distinguishing - Sandbox cannot authenticate to supervisor: check the private channel volume, matching user namespace mappings, and shared SELinux label. - Supervisor cannot call back: inspect its configured gateway endpoint, - credentials, Podman network, and gateway callback listener. + credentials, host network, and gateway callback listener. - DNS or egress denied: inspect supervisor policy decisions. Do not add a workload network, resolver bypass, or direct gateway route. - Pair is not Ready: check the supervisor health socket and gateway session. diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 0768b9c712..5588a81d81 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -18,7 +18,7 @@ identity, DNS, TCP, and loopback-forwarding semantics. | UID/GID | Pinned non-root workload identity | Same mapped identity | | Capabilities | Drop all; add none | Drop all; add none | | Seccomp | Runtime default plus sandbox-installed filters | Runtime default | -| Network | `none`; loopback only | Configured Podman network | +| Network | `none`; loopback only | Podman host network | | Gateway JWT and upstream credentials | Never mounted | Podman secrets | | User volumes and CDI devices | Workload only | Never mounted | | Channel | Private named volume, writable | Same volume, read-only | @@ -112,7 +112,7 @@ root cannot be replaced. User-owned volumes are never created or deleted. See [gateway configuration](../../docs/reference/gateway-config.mdx) for operator settings and [NETWORKING.md](NETWORKING.md) for callback networking. -The configured network and upstream proxy belong to the supervisor. +The supervisor uses Podman's host network and owns the upstream proxy settings. `health_check_interval_secs=0` uses a one-second check rather than disabling the readiness check required by this topology. diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 839a9f1bd3..da9a1c110e 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1483,6 +1483,11 @@ pub fn build_isolation_specs( supervisor.cap_drop = vec!["ALL".into()]; supervisor.cap_add.clear(); supervisor.seccomp_profile_path.clear(); + // The trusted supervisor originates approved egress from the Podman host + // network. The workload remains fenced by network=none. + supervisor.netns.nsmode = "host".into(); + supervisor.networks.clear(); + supervisor.portmappings.clear(); supervisor.devices = None; supervisor.image_volumes.clear(); supervisor.volumes = vec![NamedVolume { @@ -1689,6 +1694,9 @@ mod tests { assert_eq!(specs.workload.netns.nsmode, "none"); assert!(specs.workload.networks.is_empty()); assert!(specs.workload.portmappings.is_empty()); + assert_eq!(specs.supervisor.netns.nsmode, "host"); + assert!(specs.supervisor.networks.is_empty()); + assert!(specs.supervisor.portmappings.is_empty()); assert!(specs.workload.env.is_empty()); assert_eq!(specs.workload.unsetenv, vec!["LD_PRELOAD", "HTTP_PROXY"]); assert!(specs.workload.secrets.is_empty()); diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 67a45a3ae1..9f6bad50da 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -251,6 +251,8 @@ namespace roots. These checks do not make host bind mounts safe. The gateway talks to the Podman API socket. The Podman driver requires Podman 5.x, cgroups v2, rootless networking, and an active Podman user socket. When `socket_path` is not set, the driver probes known socket paths, then uses the `podman` CLI to resolve the active native or machine-backed connection. It fails to start if neither method finds a socket. +The agent workload uses `network=none`. Its trusted supervisor companion uses Podman's host network for gateway callbacks and policy-approved upstream connections. + For maintainer-level implementation details, refer to the [Podman driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/README.md) and [Podman networking notes](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/NETWORKING.md). Select Podman with `compute_drivers = ["podman"]` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `sandbox_ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. From 25ea6d684fffc3db8b686c038a503b9215a8917d Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 11 Sep 2026 13:10:29 -0700 Subject: [PATCH 06/19] feat(podman): split sandbox and supervisor images Signed-off-by: Drew Newberry --- crates/openshell-driver-podman/README.md | 11 +- crates/openshell-driver-podman/src/config.rs | 6 +- .../openshell-driver-podman/src/container.rs | 20 +-- crates/openshell-driver-podman/src/driver.rs | 119 ++++++++++++------ crates/openshell-driver-podman/src/main.rs | 9 +- deploy/rpm/CONFIGURATION.md | 4 +- docs/reference/gateway-config.mdx | 12 +- docs/reference/sandbox-compute-drivers.mdx | 8 +- e2e/configs/gateway/podman.toml | 1 + e2e/with-podman-gateway.sh | 59 ++++++++- 10 files changed, 187 insertions(+), 62 deletions(-) diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 5588a81d81..3fbc264de3 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -77,11 +77,12 @@ supplementary groups. Root and unresolved identities fail before provisioning. Images must not prepopulate the reserved `/.openshell` hierarchy; this prevents image-controlled symlinks from aliasing private control state into user mounts. -The trusted runtime image supplies `/openshell-sandbox` and -`/openshell-supervisor`. Podman's read-only image volume delivers the sandbox -binary; user-namespace modes that cannot use image volumes retain the existing -trusted binary extraction path. Image and request environment belong to agent -children, never the supervisor process. +`sandbox_runtime_image` supplies the statically linked musl +`/openshell-sandbox` binary. Podman's read-only image volume delivers it to the +workload; user-namespace modes that cannot use image volumes retain the trusted +binary extraction path. `supervisor_image` supplies the dynamically linked +glibc `/openshell-supervisor` binary outside the workload. Image and request +environment belong to agent children, never the supervisor process. ## Lifecycle and readiness diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 9f139cde94..1d47434b1e 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -60,9 +60,11 @@ pub struct PodmanComputeConfig { pub host_gateway_ip: String, /// Container stop timeout in seconds (SIGTERM → SIGKILL). pub stop_timeout_secs: u32, - /// OCI image containing the openshell-sandbox supervisor binary. + /// OCI image containing the statically linked `openshell-sandbox` binary. /// Mounted read-only into sandbox containers at /opt/openshell/bin /// using Podman's `type=image` mount. + pub sandbox_runtime_image: String, + /// OCI image containing the dynamically linked `openshell-supervisor` binary. pub supervisor_image: String, /// Host path to the CA certificate for sandbox mTLS. /// @@ -469,6 +471,7 @@ impl Default for PodmanComputeConfig { network_name: DEFAULT_NETWORK_NAME.to_string(), host_gateway_ip: Self::default_host_gateway_ip(), stop_timeout_secs: DEFAULT_PODMAN_STOP_TIMEOUT_SECS, + sandbox_runtime_image: openshell_core::config::default_sandbox_runtime_image(), supervisor_image: openshell_core::config::default_supervisor_image(), guest_tls_ca: None, guest_tls_cert: None, @@ -503,6 +506,7 @@ impl std::fmt::Debug for PodmanComputeConfig { .field("network_name", &self.network_name) .field("host_gateway_ip", &self.host_gateway_ip) .field("stop_timeout_secs", &self.stop_timeout_secs) + .field("sandbox_runtime_image", &self.sandbox_runtime_image) .field("supervisor_image", &self.supervisor_image) .field("guest_tls_ca", &self.guest_tls_ca) .field("guest_tls_cert", &self.guest_tls_cert) diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index da9a1c110e..159d70f60f 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1097,7 +1097,7 @@ fn build_base_spec( Vec::new() } else { vec![ImageVolume { - source: config.supervisor_image.clone(), + source: config.sandbox_runtime_image.clone(), destination: SUPERVISOR_MOUNT_DIR.into(), rw: false, }] @@ -1115,10 +1115,10 @@ fn build_base_spec( labels, env, volumes, - // Side-load the supervisor binary from a standalone OCI image. + // Side-load the sandbox runtime binary from its standalone OCI image. // Podman resolves image_volumes at the libpod layer, mounting the // image's filesystem at the destination path without starting a - // container from it. The supervisor image exposes the binary at + // container from it. The sandbox runtime image exposes the binary at // /openshell-sandbox, so it appears at /opt/openshell/bin/openshell-sandbox. image_volumes, hostname: format!("sandbox-{}", sandbox.name), @@ -2593,7 +2593,7 @@ mod tests { } #[test] - fn container_spec_includes_supervisor_image_volume() { + fn container_spec_includes_sandbox_runtime_image_volume() { let sandbox = test_sandbox("test-id", "test-name"); let config = test_config(); let spec = build_container_spec(&sandbox, &config); @@ -2610,8 +2610,8 @@ mod tests { let vol = &image_volumes[0]; assert_eq!( vol["source"].as_str(), - Some(openshell_core::config::default_supervisor_image().as_str()), - "image volume source should be the supervisor image" + Some(openshell_core::config::default_sandbox_runtime_image().as_str()), + "image volume source should be the sandbox runtime image" ); assert_eq!( vol["destination"].as_str(), @@ -2696,9 +2696,9 @@ mod tests { let image_volumes = spec["image_volumes"] .as_array() .expect("image_volumes should be an array"); - let expected_supervisor = openshell_core::config::default_supervisor_image(); + let expected_sandbox_runtime = openshell_core::config::default_sandbox_runtime_image(); assert!(image_volumes.iter().any(|volume| { - volume["source"].as_str() == Some(expected_supervisor.as_str()) + volume["source"].as_str() == Some(expected_sandbox_runtime.as_str()) && volume["destination"].as_str() == Some("/opt/openshell/bin") })); assert!(image_volumes.iter().any(|volume| { @@ -3505,7 +3505,7 @@ mod tests { !image_volumes .iter() .any(|v| v["destination"].as_str() == Some(SUPERVISOR_MOUNT_DIR)), - "supervisor image volume should not be present when bind path is provided" + "sandbox runtime image volume should not be present when bind path is provided" ); let mounts = spec["mounts"] @@ -3539,7 +3539,7 @@ mod tests { image_volumes .iter() .any(|v| v["destination"].as_str() == Some(SUPERVISOR_MOUNT_DIR)), - "supervisor image volume should be present by default" + "sandbox runtime image volume should be present by default" ); let mounts = spec["mounts"] diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 9270d6bf35..722dabf504 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -13,7 +13,7 @@ use crate::watcher::{ use openshell_core::ComputeDriverError; use openshell_core::config::CDI_GPU_DEVICE_ALL; use openshell_core::driver_utils::{ - GatewayCallbackRoute, SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, + GatewayCallbackRoute, SANDBOX_RUNTIME_IMAGE_BINARY_PATH, extract_first_tar_entry, gateway_callback_endpoint, supervisor_image_should_refresh, temp_extract_container_name, validate_linux_elf_binary, write_cache_binary_atomic, }; @@ -798,10 +798,25 @@ impl PodmanComputeDriver { let (image, immutable_image_id, image_user, image_env) = async { let phase_status = openshell_otel::ErrorStatusGuard::current(); let result = async { - // The supervisor binary is shipped in a standalone OCI image and - // mounted into sandbox containers via Podman's type=image mount. + // The sandbox runtime is shipped in a standalone OCI image and + // mounted into workload containers via Podman's type=image mount. + let sandbox_runtime_pull_policy = + runtime_image_pull_policy(&self.config.sandbox_runtime_image); + info!( + image = %self.config.sandbox_runtime_image, + policy = sandbox_runtime_pull_policy, + "Ensuring sandbox runtime image" + ); + self.client + .pull_image( + &self.config.sandbox_runtime_image, + sandbox_runtime_pull_policy, + ) + .await + .map_err(ComputeDriverError::from)?; + let supervisor_pull_policy = - supervisor_image_pull_policy(&self.config.supervisor_image); + runtime_image_pull_policy(&self.config.supervisor_image); info!( image = %self.config.supervisor_image, policy = supervisor_pull_policy, @@ -880,6 +895,16 @@ impl PodmanComputeDriver { .await?; let channel_volume = crate::isolation::channel_volume_name(&sandbox.id); let mut runtime_config = self.config.clone(); + runtime_config.sandbox_runtime_image = self + .client + .inspect_image(&self.config.sandbox_runtime_image) + .await? + .id; + if runtime_config.sandbox_runtime_image.is_empty() { + return Err(ComputeDriverError::Precondition( + "sandbox runtime image inspection returned no immutable image ID".into(), + )); + } runtime_config.supervisor_image = self .client .inspect_image(&self.config.supervisor_image) @@ -963,7 +988,7 @@ impl PodmanComputeDriver { }; let supervisor_bin_path = if userns_needs_extraction(self.config.userns.as_deref()) { - match extract_supervisor_bin(&self.client, &runtime_config).await { + match extract_sandbox_bin(&self.client, &runtime_config).await { Ok(path) => Some(path), Err(e) => { cleanup_created().await; @@ -1635,7 +1660,7 @@ fn validate_apparmor_support( Ok(()) } -fn supervisor_image_pull_policy(image: &str) -> &'static str { +fn runtime_image_pull_policy(image: &str) -> &'static str { if supervisor_image_should_refresh(image) { "newer" } else { @@ -1707,34 +1732,37 @@ fn validate_rootless_local_callback_helper( ))) } -// ── Supervisor binary extraction (userns fallback) ───────────────────── +// ── Sandbox binary extraction (userns fallback) ──────────────────────── -async fn extract_supervisor_bin( +async fn extract_sandbox_bin( client: &PodmanClient, config: &PodmanComputeConfig, ) -> Result { let mut inspect = client - .inspect_image(&config.supervisor_image) + .inspect_image(&config.sandbox_runtime_image) .await .map_err(ComputeDriverError::from)?; - if supervisor_image_should_refresh(&config.supervisor_image) { + if supervisor_image_should_refresh(&config.sandbox_runtime_image) { info!( - image = %config.supervisor_image, - "Refreshing mutable podman supervisor image" + image = %config.sandbox_runtime_image, + "Refreshing mutable Podman sandbox runtime image" ); - match client.pull_image(&config.supervisor_image, "always").await { + match client + .pull_image(&config.sandbox_runtime_image, "always") + .await + { Ok(()) => { inspect = client - .inspect_image(&config.supervisor_image) + .inspect_image(&config.sandbox_runtime_image) .await .map_err(ComputeDriverError::from)?; } Err(err) => { warn!( - image = %config.supervisor_image, + image = %config.sandbox_runtime_image, error = %err, - "Failed to refresh mutable podman supervisor image; \ + "Failed to refresh mutable Podman sandbox runtime image; \ falling back to local image if present", ); } @@ -1743,36 +1771,35 @@ async fn extract_supervisor_bin( let digest = if inspect.id.is_empty() { return Err(ComputeDriverError::Precondition(format!( - "supervisor image '{}' has no ID", - config.supervisor_image, + "sandbox runtime image '{}' has no ID", + config.sandbox_runtime_image, ))); } else { &inspect.id }; - let cache_path = - openshell_core::driver_utils::supervisor_cache_path("podman-supervisor", digest) - .map_err(ComputeDriverError::Precondition)?; + let cache_path = openshell_core::driver_utils::supervisor_cache_path("podman-sandbox", digest) + .map_err(ComputeDriverError::Precondition)?; if cache_path.is_file() { validate_linux_elf_binary(&cache_path).map_err(ComputeDriverError::Precondition)?; info!( cache_path = %cache_path.display(), - "Using cached supervisor binary" + "Using cached sandbox binary" ); return Ok(cache_path); } info!( - image = %config.supervisor_image, + image = %config.sandbox_runtime_image, cache_path = %cache_path.display(), - "Extracting supervisor binary from image" + "Extracting sandbox binary from image" ); let container_name = temp_extract_container_name(); let spec = serde_json::json!({ - "image": config.supervisor_image, + "image": config.sandbox_runtime_image, "name": container_name, - "entrypoint": [SUPERVISOR_IMAGE_BINARY_PATH], + "entrypoint": [SANDBOX_RUNTIME_IMAGE_BINARY_PATH], "command": [], }); client @@ -1786,7 +1813,7 @@ async fn extract_supervisor_bin( warn!( container = container_name, error = %err, - "Failed to remove supervisor extractor container" + "Failed to remove sandbox runtime extractor container" ); } @@ -1799,13 +1826,13 @@ async fn extract_binary_from_container( cache_path: &Path, ) -> Result { let tar_bytes = client - .copy_from_container(container_name, SUPERVISOR_IMAGE_BINARY_PATH) + .copy_from_container(container_name, SANDBOX_RUNTIME_IMAGE_BINARY_PATH) .await .map_err(ComputeDriverError::from)?; let binary_bytes = extract_first_tar_entry(&tar_bytes).map_err(|err| { ComputeDriverError::Precondition(format!( - "failed to extract supervisor binary from tar: {err}" + "failed to extract sandbox binary from tar: {err}" )) })?; @@ -2179,8 +2206,13 @@ mod tests { let subscriber = tracing_subscriber::registry().with(crate::otel_tracing::TRACING.layer(&provider)); + let mut sandbox = plain_sandbox("sandbox-trace", "demo"); + sandbox.spec = Some(DriverSandboxSpec { + launch_authentication: encoded_launch_authentication(), + ..DriverSandboxSpec::default() + }); test_driver(socket_path.clone()) - .create_sandbox(&plain_sandbox("sandbox-trace", "demo")) + .create_sandbox(&sandbox) .with_subscriber(subscriber) .await .expect("create should succeed"); @@ -2948,25 +2980,25 @@ mod tests { #[test] fn supervisor_pull_policy_refreshes_mutable_tags_only() { assert_eq!( - supervisor_image_pull_policy("ghcr.io/nvidia/openshell/supervisor:dev"), + runtime_image_pull_policy("ghcr.io/nvidia/openshell/supervisor:dev"), "newer" ); assert_eq!( - supervisor_image_pull_policy("ghcr.io/nvidia/openshell/supervisor:latest"), + runtime_image_pull_policy("ghcr.io/nvidia/openshell/supervisor:latest"), "newer" ); assert_eq!( - supervisor_image_pull_policy("ghcr.io/nvidia/openshell/supervisor"), + runtime_image_pull_policy("ghcr.io/nvidia/openshell/supervisor"), "newer" ); assert_eq!( - supervisor_image_pull_policy( + runtime_image_pull_policy( "ghcr.io/nvidia/openshell/supervisor:0.0.47-dev.13-g57b71c68f" ), "missing" ); assert_eq!( - supervisor_image_pull_policy("ghcr.io/nvidia/openshell/supervisor@sha256:abc123"), + runtime_image_pull_policy("ghcr.io/nvidia/openshell/supervisor@sha256:abc123"), "missing" ); } @@ -3358,6 +3390,7 @@ mod tests { fn create_setup_responses(proxy_secret: bool) -> Vec { let mut responses = vec![ + StubResponse::new(StatusCode::OK, "{}"), // sandbox runtime pull StubResponse::new(StatusCode::OK, "{}"), // supervisor pull StubResponse::new(StatusCode::OK, "{}"), // workload pull image_response("sha256:sandbox"), @@ -3366,6 +3399,7 @@ mod tests { StubResponse::new(StatusCode::NOT_FOUND, ""), // optional passwd StubResponse::new(StatusCode::NOT_FOUND, ""), // optional group StubResponse::new(StatusCode::NO_CONTENT, ""), // remove stopped reader + image_response("sha256:sandbox-runtime"), image_response("sha256:supervisor"), StubResponse::new(StatusCode::CREATED, "{}"), // workspace volume ]; @@ -3404,6 +3438,7 @@ mod tests { let (path, requests, handle) = spawn_podman_stub( "reserved-control-root", vec![ + StubResponse::new(StatusCode::OK, "{}"), StubResponse::new(StatusCode::OK, "{}"), StubResponse::new(StatusCode::OK, "{}"), image_response("sha256:image"), @@ -3448,9 +3483,14 @@ mod tests { .collect(), ); let driver = test_driver_with_config(proxy_auth_config(socket_path.clone(), &auth_file)); + let mut sandbox = plain_sandbox(sandbox_id, "demo"); + sandbox.spec = Some(DriverSandboxSpec { + launch_authentication: encoded_launch_authentication(), + ..DriverSandboxSpec::default() + }); driver - .create_sandbox(&plain_sandbox(sandbox_id, "demo")) + .create_sandbox(&sandbox) .await .expect_err("container create should fail"); @@ -3489,9 +3529,14 @@ mod tests { .collect(), ); let driver = test_driver_with_config(proxy_auth_config(socket_path.clone(), &auth_file)); + let mut sandbox = plain_sandbox(sandbox_id, "demo"); + sandbox.spec = Some(DriverSandboxSpec { + launch_authentication: encoded_launch_authentication(), + ..DriverSandboxSpec::default() + }); driver - .create_sandbox(&plain_sandbox(sandbox_id, "demo")) + .create_sandbox(&sandbox) .await .expect_err("container start should fail"); diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index e4554602f4..62a2dde5a1 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -101,7 +101,11 @@ struct Args { )] health_check_interval_secs: Option, - /// OCI image containing the openshell-sandbox supervisor binary. + /// OCI image containing the `openshell-sandbox` runtime binary. + #[arg(long, env = "OPENSHELL_SANDBOX_RUNTIME_IMAGE")] + sandbox_runtime_image: Option, + + /// OCI image containing the `openshell-supervisor` control binary. #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] supervisor_image: Option, @@ -207,6 +211,9 @@ async fn main() -> Result<()> { ssh_socket_path: args.sandbox_ssh_socket_path, network_name: args.network_name, stop_timeout_secs: args.stop_timeout, + sandbox_runtime_image: args + .sandbox_runtime_image + .unwrap_or_else(openshell_core::config::default_sandbox_runtime_image), supervisor_image: args .supervisor_image .unwrap_or_else(openshell_core::config::default_supervisor_image), diff --git a/deploy/rpm/CONFIGURATION.md b/deploy/rpm/CONFIGURATION.md index e5b16d00d0..af2e97a94f 100644 --- a/deploy/rpm/CONFIGURATION.md +++ b/deploy/rpm/CONFIGURATION.md @@ -222,7 +222,8 @@ overrides that persist across package upgrades. | `bind_address` | `127.0.0.1:17670` (gateway default) | Address for the primary gRPC/HTTP API listener. | | `compute_driver` | `"podman"` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman; legacy `compute_drivers` lists are rejected. | | `[openshell.drivers.podman].default_image` | `ghcr.io/nvidia/openshell-community/sandboxes/base:latest` | Default sandbox image. | -| `[openshell.drivers.podman].supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Supervisor image mounted into Podman sandboxes. | +| `[openshell.drivers.podman].sandbox_runtime_image` | `ghcr.io/nvidia/openshell/sandbox:latest` | Static musl sandbox runtime image mounted into Podman workloads. | +| `[openshell.drivers.podman].supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Dynamic glibc supervisor image used outside the workload. | | `[openshell.gateway].guest_tls_ca`, `guest_tls_cert`, `guest_tls_key` | auto-generated paths | Gateway-owned client TLS material injected into the selected local driver and mounted into sandbox containers. | | `[openshell.gateway.tls]` paths | auto-generated paths | Server TLS certificate, key, and client CA. | | `disable_tls` | unset | Set to `true` to disable TLS. | @@ -270,6 +271,7 @@ To pin specific image versions instead of `:latest`, set these values in `[openshell.drivers.podman]`: ```toml +sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:v0.0.37" supervisor_image = "ghcr.io/nvidia/openshell/supervisor:v0.0.37" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:v0.0.37" ``` diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index e23137827e..e60d576f97 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -532,6 +532,10 @@ service_account_name = "openshell-sandbox" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" image_pull_policy = "IfNotPresent" image_pull_secrets = ["regcred"] +# Statically linked musl workload-side runtime. +# sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:" +sandbox_runtime_image_pull_policy = "IfNotPresent" +# Dynamically linked glibc control-side runtime. # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" supervisor_image_pull_policy = "IfNotPresent" @@ -643,9 +647,9 @@ sandbox_label = "docker-dev" # Optional override. When omitted, the gateway derives # https://host.openshell.internal: for this driver. grpc_endpoint = "https://host.openshell.internal:17670" -# The workload runtime and supervisor companion use separate images. Both -# default to the gateway version; override either to pin a specific build. +# Statically linked musl workload-side runtime. Defaults to the gateway version. # sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:" +# Dynamically linked glibc control-side runtime. Defaults to the gateway version. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" @@ -704,7 +708,9 @@ network_name = "openshell" # host_gateway_ip = "192.168.127.254" sandbox_ssh_socket_path = "/run/openshell/ssh.sock" stop_timeout_secs = 45 -# Defaults to the gateway version; override to pin a specific build. +# Statically linked musl workload-side runtime. Defaults to the gateway version. +# sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:" +# Dynamically linked glibc control-side runtime. Defaults to the gateway version. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 9f6bad50da..691f7077ac 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -175,7 +175,7 @@ that already covers loopback. Otherwise, the Docker driver requests a separate For maintainer-level implementation details, refer to the [Docker driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-docker/README.md). -Select Docker with `compute_drivers = ["docker"]` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_runtime_image`, `supervisor_image`, `image_pull_policy`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. The sandbox runtime image contains `/openshell-sandbox`; the supervisor companion image contains `/openshell-supervisor`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. +Select Docker with `compute_drivers = ["docker"]` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_runtime_image`, `supervisor_image`, `image_pull_policy`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. The sandbox runtime image contains the static musl `/openshell-sandbox` binary; the supervisor image contains the dynamic glibc `/openshell-supervisor` binary. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. When operating `openshell-driver-docker` as an external driver, set `OPENSHELL_OTLP_ENDPOINT` to export its spans. The driver continues W3C trace @@ -255,7 +255,7 @@ The agent workload uses `network=none`. Its trusted supervisor companion uses Po For maintainer-level implementation details, refer to the [Podman driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/README.md) and [Podman networking notes](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/NETWORKING.md). -Select Podman with `compute_drivers = ["podman"]` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `sandbox_ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. +Select Podman with `compute_drivers = ["podman"]` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `sandbox_runtime_image`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `sandbox_ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. Podman sandboxes default to a 45-second graceful stop window before Podman escalates from `SIGTERM` to `SIGKILL`. Set `stop_timeout_secs` in gateway config, or `OPENSHELL_STOP_TIMEOUT` for the standalone driver, when a local runtime needs a different teardown window. @@ -423,7 +423,9 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `[managed_ssh_ingress]` | `networkPolicy.enabled` | In managed mode, create an SSH ingress policy in every workspace namespace. Helm configures the gateway namespace and pod selector automatically. Operator mode leaves namespace policy management to the platform operator. | | `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. | | `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. | -| `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the trusted runtime image that provides the `openshell-sandbox` and `openshell-supervisor` binaries. The default repository with an empty tag uses the version pinned into the gateway. | +| `sandbox_runtime_image` | `sandboxRuntime.image.repository` / `sandboxRuntime.image.tag` | Override the image that provides the static musl `openshell-sandbox` binary. The default repository with an empty tag uses the version pinned into the gateway. | +| `sandbox_runtime_image_pull_policy` | `sandboxRuntime.image.pullPolicy` | Set the Kubernetes image pull policy for the sandbox runtime image. | +| `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the image that provides the dynamic glibc `openshell-supervisor` binary. The default repository with an empty tag uses the version pinned into the gateway. | | `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | | `sandbox_runtime.network_policy_enforced` | `supervisor.sandboxRuntime.networkPolicyEnforced` | Acknowledge that the cluster CNI enforces ingress and egress `NetworkPolicy` in sandbox namespaces. This must be `true`. | | `sandbox_runtime.boundary_port` | `supervisor.sandboxRuntime.boundaryPort` | Set the non-privileged TLS port used between the paired supervisor and sandbox Pods. | diff --git a/e2e/configs/gateway/podman.toml b/e2e/configs/gateway/podman.toml index 6eff3b5615..a53708c61a 100644 --- a/e2e/configs/gateway/podman.toml +++ b/e2e/configs/gateway/podman.toml @@ -26,5 +26,6 @@ health_check_interval_secs = 10 network_name = "openshell-e2e" grpc_endpoint = "http://host.containers.internal:8080" ssh_socket_path = "/run/openshell/ssh.sock" +sandbox_runtime_image = "localhost/openshell/sandbox:e2e-vm" supervisor_image = "localhost/openshell/supervisor:e2e-vm" app_armor_profile = "Unconfined" diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index e20247cb6a..f7f3d48401 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -354,6 +354,26 @@ resolve_podman_supervisor_image() { printf '%s\n' "openshell/supervisor:dev" } +resolve_podman_sandbox_runtime_image() { + if [ -n "${OPENSHELL_SANDBOX_RUNTIME_IMAGE:-}" ]; then + printf '%s\n' "${OPENSHELL_SANDBOX_RUNTIME_IMAGE}" + return 0 + fi + + if [ -n "${CI:-}" ]; then + if [ -z "${IMAGE_TAG:-}" ]; then + echo "ERROR: IMAGE_TAG must be set in CI when no Podman sandbox runtime image override is provided." >&2 + exit 2 + fi + + local registry="${OPENSHELL_REGISTRY:-ghcr.io/nvidia/openshell}" + printf '%s/sandbox:%s\n' "${registry%/}" "${IMAGE_TAG}" + return 0 + fi + + printf '%s\n' "openshell/sandbox:dev" +} + ensure_podman_supervisor_image() { local image=$1 @@ -448,6 +468,37 @@ ensure_podman_supervisor_image() { exit 2 } +ensure_podman_sandbox_runtime_image() { + local image=$1 + + if [ "${image}" = "openshell/sandbox:dev" ] \ + && [ -z "${OPENSHELL_SANDBOX_RUNTIME_IMAGE:-}" ] \ + && [ -z "${CI:-}" ]; then + echo "Building local Podman sandbox runtime image ${image}..." + with_podman_config env CONTAINER_ENGINE=podman IMAGE_TAG=dev \ + bash "${ROOT}/tasks/scripts/docker-build-image.sh" sandbox + if podman_cmd image exists "${image}" 2>/dev/null; then + return 0 + fi + + echo "ERROR: expected sandbox runtime image '${image}' after local build." >&2 + exit 2 + fi + + if podman_cmd image exists "${image}" 2>/dev/null; then + return 0 + fi + + echo "Pulling Podman sandbox runtime image ${image}..." + if podman_cmd pull "${image}"; then + return 0 + fi + + echo "ERROR: sandbox runtime image '${image}' is not available." >&2 + echo " Build it, push it, or set OPENSHELL_SANDBOX_RUNTIME_IMAGE to a pullable image." >&2 + exit 2 +} + if [ -n "${OPENSHELL_GATEWAY_ENDPOINT:-}" ]; then case "${OPENSHELL_GATEWAY_ENDPOINT}" in http://*) ;; @@ -542,6 +593,10 @@ podman_cmd run --rm --network none --entrypoint /sbin/apk \ SUPERVISOR_PACKAGE_MANIFEST_SHA256="$(sha256sum "${SUPERVISOR_PACKAGE_MANIFEST}" | cut -d' ' -f1)" echo "Using Podman supervisor image: ${SUPERVISOR_RUNTIME_IMAGE} (ID ${SUPERVISOR_IMAGE_ID}, digest ${SUPERVISOR_IMAGE_DIGEST}, base ${SUPERVISOR_BASE_IMAGE} ID ${SUPERVISOR_BASE_IMAGE_ID} digest ${SUPERVISOR_BASE_IMAGE_DIGEST}, packages ${SUPERVISOR_PACKAGE_MANIFEST_SHA256})" +SANDBOX_RUNTIME_IMAGE="$(resolve_podman_sandbox_runtime_image)" +ensure_podman_sandbox_runtime_image "${SANDBOX_RUNTIME_IMAGE}" +echo "Using Podman sandbox runtime image: ${SANDBOX_RUNTIME_IMAGE}" + DEFAULT_SANDBOX_IMAGE="ghcr.io/nvidia/openshell-community/sandboxes/base:latest" SANDBOX_IMAGE_REQUEST="${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-${OPENSHELL_SANDBOX_IMAGE:-${DEFAULT_SANDBOX_IMAGE}}}" if [ "${OPENSHELL_E2E_REQUIRE_DIGEST_PINNED_SANDBOX_IMAGE:-0}" = "1" ] \ @@ -666,7 +721,7 @@ if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then driver_tls_ca_sha256="$(sha256sum "${EXTERNAL_DRIVER_TLS_CA}" | cut -d' ' -f1)" driver_tls_cert_sha256="$(sha256sum "${EXTERNAL_DRIVER_TLS_CERT}" | cut -d' ' -f1)" driver_tls_key_sha256="$(sha256sum "${EXTERNAL_DRIVER_TLS_KEY}" | cut -d' ' -f1)" - external_driver_environment="$(printf '{\"OPENSHELL_COMPUTE_DRIVER_SOCKET\":\"%s\",\"OPENSHELL_PODMAN_SOCKET\":\"%s\",\"OPENSHELL_SANDBOX_IMAGE\":\"%s\",\"OPENSHELL_SANDBOX_IMAGE_PULL_POLICY\":\"%s\",\"OPENSHELL_HEALTH_CHECK_INTERVAL_SECS\":%s,\"OPENSHELL_GRPC_ENDPOINT\":\"%s\",\"OPENSHELL_GATEWAY_PORT\":%s,\"OPENSHELL_NETWORK_NAME\":\"%s\",\"OPENSHELL_STOP_TIMEOUT\":%s,\"OPENSHELL_SUPERVISOR_IMAGE\":\"%s\",\"OPENSHELL_PODMAN_TLS_CA\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_PODMAN_TLS_CERT\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_PODMAN_TLS_KEY\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_ENABLE_BIND_MOUNTS\":%s}' \ + external_driver_environment="$(printf '{\"OPENSHELL_COMPUTE_DRIVER_SOCKET\":\"%s\",\"OPENSHELL_PODMAN_SOCKET\":\"%s\",\"OPENSHELL_SANDBOX_IMAGE\":\"%s\",\"OPENSHELL_SANDBOX_IMAGE_PULL_POLICY\":\"%s\",\"OPENSHELL_HEALTH_CHECK_INTERVAL_SECS\":%s,\"OPENSHELL_GRPC_ENDPOINT\":\"%s\",\"OPENSHELL_GATEWAY_PORT\":%s,\"OPENSHELL_NETWORK_NAME\":\"%s\",\"OPENSHELL_STOP_TIMEOUT\":%s,\"OPENSHELL_SANDBOX_RUNTIME_IMAGE\":\"%s\",\"OPENSHELL_SUPERVISOR_IMAGE\":\"%s\",\"OPENSHELL_PODMAN_TLS_CA\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_PODMAN_TLS_CERT\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_PODMAN_TLS_KEY\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_ENABLE_BIND_MOUNTS\":%s}' \ "${DRIVER_SOCKET}" \ "${OPENSHELL_PODMAN_SOCKET:-}" \ "${SANDBOX_RUNTIME_IMAGE}" \ @@ -676,6 +731,7 @@ if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then "${HOST_PORT}" \ "${PODMAN_NETWORK_NAME}" \ "${PODMAN_STOP_TIMEOUT_SECS}" \ + "${SANDBOX_RUNTIME_IMAGE}" \ "${SUPERVISOR_RUNTIME_IMAGE}" \ "${EXTERNAL_DRIVER_TLS_CA}" \ "${driver_tls_ca_sha256}" \ @@ -736,6 +792,7 @@ if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then OPENSHELL_GATEWAY_PORT="${HOST_PORT}" \ OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ OPENSHELL_STOP_TIMEOUT="${PODMAN_STOP_TIMEOUT_SECS}" \ + OPENSHELL_SANDBOX_RUNTIME_IMAGE="${SANDBOX_RUNTIME_IMAGE}" \ OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_RUNTIME_IMAGE}" \ OPENSHELL_PODMAN_TLS_CA="${EXTERNAL_DRIVER_TLS_CA}" \ OPENSHELL_PODMAN_TLS_CERT="${EXTERNAL_DRIVER_TLS_CERT}" \ From 5693eec695168029079a486baad232848aa8ce14 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 11 Sep 2026 15:10:16 -0700 Subject: [PATCH 07/19] fix(podman): repair rebase integration Signed-off-by: Drew Newberry --- .../openshell-driver-podman/src/container.rs | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 159d70f60f..a8bdfcca19 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -534,7 +534,7 @@ fn build_env( ); env.insert( openshell_core::sandbox_env::SSH_SOCKET_PATH.into(), - config.sandbox_ssh_socket_path.clone(), + config.ssh_socket_path.clone(), ); env.insert("OPENSHELL_CONTAINER_IMAGE".into(), image.to_string()); let main_process = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(spec) @@ -664,8 +664,8 @@ fn build_resource_limits(sandbox: &DriverSandbox, config: &PodmanComputeConfig) } } -fn podman_pids_limit(value: i64) -> Option { - if value > 0 { Some(value) } else { None } +fn podman_pids_limit(value: Option) -> Option { + value.map(std::num::NonZeroI64::get) } pub fn podman_driver_volume_mount_sources( @@ -1150,11 +1150,14 @@ fn build_base_spec( "CMD-SHELL".into(), format!( "test -e /var/run/openshell-ssh-ready || test -S {} || ss -tlnp | grep -q :{}", - config.sandbox_ssh_socket_path, + config.ssh_socket_path, openshell_core::config::DEFAULT_SSH_PORT ), ], - interval: config.health_check_interval_secs * 1_000_000_000, + interval: config + .health_check_interval_secs + .map_or(10, std::num::NonZeroU64::get) + * 1_000_000_000, timeout: 2_000_000_000, retries: 10, start_period: 5_000_000_000, @@ -1526,8 +1529,11 @@ pub fn build_isolation_specs( "--socket".into(), "/run/openshell/supervisor-health.sock".into(), ]; - supervisor.healthconfig.interval = - input.config.health_check_interval_secs.max(1) * 1_000_000_000; + supervisor.healthconfig.interval = input + .config + .health_check_interval_secs + .map_or(10, std::num::NonZeroU64::get) + * 1_000_000_000; Ok(IsolationSpecs { workload, supervisor, @@ -1794,7 +1800,7 @@ mod tests { ); assert_eq!( spec["resource_limits"]["PidsLimit"].as_i64(), - Some(crate::config::DEFAULT_SANDBOX_PIDS_LIMIT) + openshell_core::config::default_sandbox_pids_limit().map(std::num::NonZeroI64::get) ); } @@ -1802,7 +1808,7 @@ mod tests { fn container_spec_can_inherit_runtime_pids_limit() { let sandbox = test_sandbox("test-id", "test-name"); let mut config = test_config(); - config.sandbox_pids_limit = 0; + config.sandbox_pids_limit = None; let spec = build_container_spec(&sandbox, &config); assert!(spec["resource_limits"].get("PidsLimit").is_none()); @@ -2161,7 +2167,7 @@ mod tests { fn container_spec_healthcheck_interval_from_config() { let sandbox = test_sandbox("test-id", "test-name"); let mut config = test_config(); - config.health_check_interval_secs = 30; + config.health_check_interval_secs = std::num::NonZeroU64::new(30); let spec = build_container_spec(&sandbox, &config); let interval = spec["healthconfig"]["Interval"] @@ -2587,7 +2593,7 @@ mod tests { default_image: "test-image:latest".to_string(), grpc_endpoint: "http://localhost:50051".to_string(), host_gateway_ip: String::new(), - sandbox_ssh_socket_path: "/run/openshell/test-ssh.sock".to_string(), + ssh_socket_path: "/run/openshell/test-ssh.sock".to_string(), ..PodmanComputeConfig::default() } } From bd652ee27ce560737b4843c756b4f85f6d456b76 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 12 Sep 2026 14:09:19 -0700 Subject: [PATCH 08/19] refactor(podman): name the sandbox runtime directly Signed-off-by: Drew Newberry --- crates/openshell-driver-podman/README.md | 4 +-- .../openshell-driver-podman/src/container.rs | 7 +++-- crates/openshell-driver-podman/src/driver.rs | 6 ++--- .../openshell-driver-podman/src/isolation.rs | 27 +++++++++---------- e2e/rust/tests/podman_oci_identity.rs | 2 +- 5 files changed, 22 insertions(+), 24 deletions(-) diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 3fbc264de3..4166c0ba71 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -57,7 +57,7 @@ namespace so the sandbox's loopback DNS relay can bind port 53 without a capability. No nftables or nested network namespace setup runs in the sandbox. The channel contains the sandbox bootstrap and sandbox-side TLS identity only. -Supervisor private keys and topology stay in the companion's private filesystem. +Supervisor private keys and the runtime descriptor stay in the supervisor's private filesystem. Landlock denies agent access to the top-level `/.openshell` control hierarchy. The driver verifies Podman's reported `network=none` fence before launch and restart. `host.containers.internal` and callback networking apply to the @@ -115,7 +115,7 @@ See [gateway configuration](../../docs/reference/gateway-config.mdx) for operator settings and [NETWORKING.md](NETWORKING.md) for callback networking. The supervisor uses Podman's host network and owns the upstream proxy settings. `health_check_interval_secs=0` uses a one-second check rather than disabling -the readiness check required by this topology. +the readiness check required by this architecture. Gateway OTLP configuration continues to export compute-driver spans under the `openshell-driver-podman` service, preserving gateway trace context. diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index a8bdfcca19..7a63f75adc 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1464,17 +1464,16 @@ pub fn build_isolation_specs( supervisor.image.clone_from(&input.config.supervisor_image); supervisor.entrypoint = vec!["/openshell-supervisor".into()]; supervisor.command.extend([ - "--topology-backend-name=podman".into(), format!( - "--topology-payload-file={}", - crate::isolation::TOPOLOGY_PATH + "--backend-descriptor-file={}", + crate::isolation::RUNTIME_DESCRIPTOR_PATH ), format!("--auth-bundle-file={}", crate::isolation::AUTH_BUNDLE_PATH), "--health-socket-path=/run/openshell/supervisor-health.sock".into(), ]); supervisor.env.insert( openshell_core::sandbox_env::ADMITTED_ISOLATION_BACKEND.into(), - "podman".into(), + openshell_sandbox_backend::BACKEND_NAME.into(), ); supervisor.user = user; supervisor.groups = input diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 722dabf504..0c97bd84b3 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -455,7 +455,7 @@ impl PodmanComputeDriver { } // Auto-detect the gRPC callback endpoint before deciding whether this - // topology needs the Podman bridge gateway address. + // callback route needs the Podman bridge gateway address. if config.grpc_endpoint.is_empty() { config.grpc_endpoint = gateway_callback_endpoint( GatewayCallbackRoute::Podman, @@ -470,7 +470,7 @@ impl PodmanComputeDriver { } // Ensure the bridge network exists. Inspect its gateway only when the - // selected Linux callback topology will bind that exact address. + // selected Linux callback route will bind that exact address. client.ensure_network(&config.network_name).await?; let uses_local_callback_alias = Url::parse(&config.grpc_endpoint) .ok() @@ -3314,7 +3314,7 @@ mod tests { ), StubResponse::new(StatusCode::OK, archive.into_inner().unwrap()), StubResponse::new(StatusCode::OK, "").with_archive_members(channel_archive_members()), - StubResponse::new(StatusCode::OK, ""), // refreshed supervisor auth and topology + StubResponse::new(StatusCode::OK, ""), // refreshed supervisor auth and runtime descriptor fence_response(), StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor start diff --git a/crates/openshell-driver-podman/src/isolation.rs b/crates/openshell-driver-podman/src/isolation.rs index 8050ae4790..09c6b5339e 100644 --- a/crates/openshell-driver-podman/src/isolation.rs +++ b/crates/openshell-driver-podman/src/isolation.rs @@ -11,7 +11,7 @@ use openshell_core::ComputeDriverError; use openshell_core::proto::compute::v1::DriverSandbox; use openshell_isolation_interface::contract::{DriverFenceEvidence, ResolvedWorkloadIdentity}; use openshell_sandbox_backend::boundary_protocol::{ - BoundaryConfig, BoundaryListener, BoundaryTopology, GatewayVerificationKey, + BoundaryConfig, BoundaryListener, GatewayVerificationKey, SandboxRuntimeDescriptor, SandboxTlsClientConfig, SandboxTlsServerConfig, SandboxTransport, generate_sandbox_tls_material, }; @@ -20,7 +20,7 @@ pub const LABEL_ROLE: &str = "openshell.io/isolation-role"; pub const WORKLOAD_FILTER: &str = "openshell.io/isolation-role=sandbox"; pub const CHANNEL_ROOT: &str = "/.openshell/channel"; pub const BOOTSTRAP_PATH: &str = "/.openshell/channel/sandbox/bootstrap.json"; -pub const TOPOLOGY_PATH: &str = "/.openshell/supervisor/topology.payload"; +pub const RUNTIME_DESCRIPTOR_PATH: &str = "/.openshell/supervisor/runtime-descriptor.json"; pub const AUTH_BUNDLE_PATH: &str = "/.openshell/supervisor/auth.json"; pub const RESTART_BUNDLE_PATH: &str = "/.openshell/supervisor/sandbox-bundle.tar"; const SOCKET_PATH: &str = "/.openshell/channel/sandbox/control.sock"; @@ -186,7 +186,7 @@ pub fn bootstrap_archives( driver_fence: driver_fence.clone(), child_env, }; - let topology = BoundaryTopology { + let runtime_descriptor = SandboxRuntimeDescriptor { boundary_id: sandbox_id.into(), generation, session_id, @@ -221,8 +221,8 @@ pub fn bootstrap_archives( supervisor.directory(".openshell", 0o755, false)?; supervisor.directory(".openshell/supervisor", 0o700, true)?; supervisor.file( - TOPOLOGY_PATH, - &serde_json::to_vec(&topology).map_err(invalid)?, + RUNTIME_DESCRIPTOR_PATH, + &serde_json::to_vec(&runtime_descriptor).map_err(invalid)?, )?; supervisor.file( AUTH_BUNDLE_PATH, @@ -414,20 +414,19 @@ mod tests { .unwrap(), ) .unwrap(); - let topology: BoundaryTopology = serde_json::from_slice( + let runtime_descriptor: SandboxRuntimeDescriptor = serde_json::from_slice( supervisor - .get(&PathBuf::from(TOPOLOGY_PATH.trim_start_matches('/'))) + .get(&PathBuf::from( + RUNTIME_DESCRIPTOR_PATH.trim_start_matches('/'), + )) .unwrap(), ) .unwrap(); - assert_eq!(config.boundary_id, topology.boundary_id); - assert_eq!(config.session_id, topology.session_id); - assert_eq!(config.driver_fence, topology.driver_fence); + assert_eq!(config.boundary_id, runtime_descriptor.boundary_id); + assert_eq!(config.session_id, runtime_descriptor.session_id); + assert_eq!(config.driver_fence, runtime_descriptor.driver_fence); assert_eq!(config.workload_identity, identity); - topology - .driver_fence - .validate_for_backend("podman") - .unwrap(); + runtime_descriptor.driver_fence.validate().unwrap(); assert_eq!( supervisor .get(&PathBuf::from(RESTART_BUNDLE_PATH.trim_start_matches('/'))) diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs index 15ea13a77f..e74888c84c 100644 --- a/e2e/rust/tests/podman_oci_identity.rs +++ b/e2e/rust/tests/podman_oci_identity.rs @@ -292,6 +292,6 @@ async fn assert_isolated_pair(image: &ImageGuard, sandbox: &SandboxGuard, contai .unwrap(); assert!(!mounts.contains("/etc/openshell/tls")); assert!(!mounts.contains("/.openshell/supervisor")); - let posture = sandbox.exec(&["sh", "-c", "set -eu; awk '/^CapEff:|^CapBnd:|^NoNewPrivs:/ {print}' /proc/self/status; test ! -r /.openshell/channel/sandbox/server.key; test ! -r /.openshell/supervisor/topology.payload"]).await.expect("workload cannot read either control credential set"); + let posture = sandbox.exec(&["sh", "-c", "set -eu; awk '/^CapEff:|^CapBnd:|^NoNewPrivs:/ {print}' /proc/self/status; test ! -r /.openshell/channel/sandbox/server.key; test ! -r /.openshell/supervisor/runtime-descriptor.json"]).await.expect("workload cannot read either control credential set"); assert!(posture.contains("0000000000000000")); } From c51d38273e4bda921ae1abe17273e5f332a96428 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 12 Sep 2026 22:57:10 -0700 Subject: [PATCH 09/19] fix(podman): provision supervisor CA runtime storage Signed-off-by: Drew Newberry --- .../openshell-driver-podman/src/container.rs | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 7a63f75adc..5fed7e2c03 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1451,6 +1451,20 @@ pub fn build_isolation_specs( workload .mounts .retain(|mount| !trusted_mount(&mount.destination)); + workload.mounts.push(Mount { + kind: "tmpfs".into(), + source: "tmpfs".into(), + destination: openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_DIR.into(), + options: vec![ + "rw".into(), + "nosuid".into(), + "nodev".into(), + format!("uid={}", input.identity.uid), + format!("gid={}", input.identity.gid), + "mode=0755".into(), + "size=1m".into(), + ], + }); workload.volumes.push(NamedVolume { name: channel.clone(), dest: crate::isolation::CHANNEL_ROOT.into(), @@ -1712,6 +1726,32 @@ mod tests { .iter() .all(|mount| !trusted_mount(&mount.destination)) ); + let supervisor_ca_mount = specs + .workload + .mounts + .iter() + .find(|mount| mount.destination == openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_DIR) + .expect("workload supervisor CA mount"); + assert_eq!(supervisor_ca_mount.kind, "tmpfs"); + assert_eq!(supervisor_ca_mount.source, "tmpfs"); + for option in [ + "rw", + "nosuid", + "nodev", + "uid=1000", + "gid=1001", + "mode=0755", + "size=1m", + ] { + assert!(supervisor_ca_mount.options.contains(&option.to_string())); + } + assert!( + specs + .workload + .mounts + .iter() + .all(|mount| mount.destination != "/run") + ); assert_eq!(specs.supervisor.secrets.len(), 1); assert_eq!(specs.supervisor.secrets[0].source, "jwt"); assert_eq!(specs.supervisor.secrets[0].uid, 1000); From 68da95d1a9ea64803c548f69d84c97805ca925ee Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 12 Sep 2026 23:39:56 -0700 Subject: [PATCH 10/19] fix(podman): address isolation review findings Signed-off-by: Drew Newberry --- crates/openshell-driver-podman/README.md | 5 +- crates/openshell-driver-podman/src/config.rs | 6 +- .../openshell-driver-podman/src/container.rs | 78 +++-- crates/openshell-driver-podman/src/driver.rs | 102 +++++-- .../openshell-driver-podman/src/isolation.rs | 58 ++-- docs/reference/gateway-config.mdx | 282 +++++++++++++----- docs/reference/sandbox-compute-drivers.mdx | 26 +- 7 files changed, 387 insertions(+), 170 deletions(-) diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 4166c0ba71..912a79ac7a 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -114,8 +114,9 @@ root cannot be replaced. User-owned volumes are never created or deleted. See [gateway configuration](../../docs/reference/gateway-config.mdx) for operator settings and [NETWORKING.md](NETWORKING.md) for callback networking. The supervisor uses Podman's host network and owns the upstream proxy settings. -`health_check_interval_secs=0` uses a one-second check rather than disabling -the readiness check required by this architecture. +Omit `health_check_interval_secs` to disable Podman's periodic health command. +Explicit zero is invalid. OpenShell still gates readiness on the supervisor's +authenticated health signal. Gateway OTLP configuration continues to export compute-driver spans under the `openshell-driver-podman` service, preserving gateway trace context. diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 1d47434b1e..8b4e7f240b 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -93,11 +93,11 @@ pub struct PodmanComputeConfig { /// Host path to a SPIFFE Workload API Unix socket exposed to sandbox /// supervisors for provider token exchange client assertions. pub provider_spiffe_workload_api_socket: Option, - /// `AppArmor` confinement requested for sandbox containers. Omission sends - /// no override and preserves Podman's runtime-selected profile. + /// `AppArmor` confinement requested for the workload container. Omission + /// sends no override and preserves Podman's runtime-selected profile. #[serde(default, skip_serializing_if = "Option::is_none")] pub app_armor_profile: Option, - /// Health check interval in seconds for sandbox containers. + /// Health check interval in seconds for supervisor containers. /// /// Podman runs the health check command at this interval to determine /// container readiness. Lower values detect readiness faster but diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 5fed7e2c03..e7ea5951da 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -219,6 +219,8 @@ pub struct ContainerSpec { cap_drop: Vec, cap_add: Vec, no_new_privileges: bool, + #[serde(skip_serializing_if = "Option::is_none")] + apparmor_profile: Option, #[serde(skip_serializing_if = "String::is_empty")] seccomp_profile_path: String, #[serde(skip_serializing_if = "BTreeMap::is_empty")] @@ -341,8 +343,13 @@ struct SecretMount { struct ResourceLimits { cpu: CpuLimits, memory: MemoryLimits, - #[serde(rename = "PidsLimit", skip_serializing_if = "Option::is_none")] - pids_limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pids: Option, +} + +#[derive(Serialize)] +struct PidsLimits { + limit: i64, } #[derive(Serialize)] @@ -660,7 +667,7 @@ fn build_resource_limits(sandbox: &DriverSandbox, config: &PodmanComputeConfig) period: DEFAULT_CPU_PERIOD, }, memory: MemoryLimits { limit: mem_bytes }, - pids_limit: podman_pids_limit(config.sandbox_pids_limit), + pids: podman_pids_limit(config.sandbox_pids_limit).map(|limit| PidsLimits { limit }), } } @@ -1141,6 +1148,7 @@ fn build_base_spec( cap_drop: vec!["ALL".into()], cap_add: Vec::new(), no_new_privileges: true, + apparmor_profile: None, // Omission selects the runtime default, never an unconfined profile. seccomp_profile_path: String::new(), sysctl: BTreeMap::new(), @@ -1437,6 +1445,12 @@ pub fn build_isolation_specs( .collect(); workload.cap_drop = vec!["ALL".into()]; workload.cap_add.clear(); + workload.apparmor_profile = input + .config + .app_armor_profile + .as_ref() + .and_then(openshell_core::config::AppArmorProfile::oci_security_opt) + .and_then(|option| option.strip_prefix("apparmor=").map(str::to_string)); workload.seccomp_profile_path.clear(); workload .sysctl @@ -1535,18 +1549,18 @@ pub fn build_isolation_specs( secret.uid = input.identity.uid; secret.gid = input.identity.gid; } - supervisor.healthconfig.test = vec![ - "CMD".into(), - "/openshell-supervisor".into(), - "health".into(), - "--socket".into(), - "/run/openshell/supervisor-health.sock".into(), - ]; - supervisor.healthconfig.interval = input - .config - .health_check_interval_secs - .map_or(10, std::num::NonZeroU64::get) - * 1_000_000_000; + if let Some(interval) = input.config.health_check_interval_secs { + supervisor.healthconfig.test = vec![ + "CMD".into(), + "/openshell-supervisor".into(), + "health".into(), + "--socket".into(), + "/run/openshell/supervisor-health.sock".into(), + ]; + supervisor.healthconfig.interval = interval.get() * 1_000_000_000; + } else { + supervisor.healthconfig.test = vec!["NONE".into()]; + } Ok(IsolationSpecs { workload, supervisor, @@ -1675,7 +1689,10 @@ mod tests { name: "agent".into(), ..Default::default() }; - let config = PodmanComputeConfig::default(); + let mut config = PodmanComputeConfig::default(); + config.app_armor_profile = Some(openshell_core::config::AppArmorProfile::Localhost( + "openshell-sandbox".into(), + )); let identity = openshell_isolation_interface::contract::ResolvedWorkloadIdentity::new( 1000, 1001, @@ -1711,6 +1728,14 @@ mod tests { assert!(spec.no_new_privileges); } assert_eq!(specs.workload.netns.nsmode, "none"); + assert_eq!( + specs.workload.apparmor_profile.as_deref(), + Some("openshell-sandbox") + ); + assert_eq!(specs.supervisor.apparmor_profile, None); + let workload_json = serde_json::to_string(&specs.workload).unwrap(); + assert!(workload_json.contains("\"apparmor_profile\":\"openshell-sandbox\"")); + assert_eq!(specs.supervisor.healthconfig.test, vec!["NONE"]); assert!(specs.workload.networks.is_empty()); assert!(specs.workload.portmappings.is_empty()); assert_eq!(specs.supervisor.netns.nsmode, "host"); @@ -1827,20 +1852,17 @@ mod tests { ..Default::default() }); let config = test_config(); - let spec = build_container_spec(&sandbox, &config); + let limits = build_resource_limits(&sandbox, &config); + assert_eq!(limits.cpu.quota, 50_000); + assert_eq!(limits.memory.limit, 2 * 1024 * 1024 * 1024); assert_eq!( - spec["resource_limits"]["cpu"]["quota"].as_u64(), - Some(50_000) - ); - assert_eq!( - spec["resource_limits"]["memory"]["limit"].as_u64(), - Some(2 * 1024 * 1024 * 1024) - ); - assert_eq!( - spec["resource_limits"]["PidsLimit"].as_i64(), + limits.pids.as_ref().map(|pids| pids.limit), openshell_core::config::default_sandbox_pids_limit().map(std::num::NonZeroI64::get) ); + let serialized = serde_json::to_string(&limits).unwrap(); + assert!(serialized.contains("\"pids\":{\"limit\":2048}")); + assert!(!serialized.contains("PidsLimit")); } #[test] @@ -1848,9 +1870,9 @@ mod tests { let sandbox = test_sandbox("test-id", "test-name"); let mut config = test_config(); config.sandbox_pids_limit = None; - let spec = build_container_spec(&sandbox, &config); + let limits = build_resource_limits(&sandbox, &config); - assert!(spec["resource_limits"].get("PidsLimit").is_none()); + assert!(limits.pids.is_none()); } #[test] diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 0c97bd84b3..f96ce723dd 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -30,6 +30,7 @@ use openshell_core::proto::compute::v1::{ GpuResourceCapabilities, GpuResourceRequirements, MemoryResourceCapabilities, ResourceCapabilities, gateway_listener_requirement::Selector, }; +use std::collections::HashMap; #[cfg(target_os = "linux")] use std::net::{IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; @@ -1044,14 +1045,7 @@ impl PodmanComputeDriver { let workload_id = self.client.create_typed_container(&specs.workload).await?; created_workload = Some(workload_id.clone()); self.client.verify_isolation_fence(&workload_id).await?; - let child_env = image_env - .iter() - .filter_map(|entry| { - entry - .split_once('=') - .map(|(key, value)| (key.into(), value.into())) - }) - .collect(); + let child_env = podman_child_environment(sandbox, &image_env); let launch_authentication = sandbox .spec .as_ref() @@ -1376,16 +1370,16 @@ impl PodmanComputeDriver { .await?; let archive = self .client - .copy_from_container(&supervisor, crate::isolation::RESTART_BUNDLE_PATH) + .copy_from_container(&supervisor, crate::isolation::RESTART_METADATA_PATH) .await?; let bundle = extract_first_tar_entry(&archive).map_err(ComputeDriverError::Precondition)?; - let previous_config = crate::isolation::boundary_config_from_channel_archive(&bundle)?; + let restart_metadata = crate::isolation::restart_metadata_from_slice(&bundle)?; let archives = crate::isolation::bootstrap_archives( sandbox_id, &container_id, - &previous_config.workload_identity, - previous_config.child_env, + &restart_metadata.workload_identity, + restart_metadata.child_env, &launch_authentication, )?; self.client @@ -1849,6 +1843,28 @@ fn userns_needs_extraction(userns: Option<&str>) -> bool { }) } +fn podman_child_environment( + sandbox: &DriverSandbox, + image_env: &[String], +) -> HashMap { + let mut environment = image_env + .iter() + .filter_map(|entry| { + entry + .split_once('=') + .map(|(key, value)| (key.to_string(), value.to_string())) + }) + .collect::>(); + if let Some(spec) = sandbox.spec.as_ref() { + if let Some(template) = spec.template.as_ref() { + environment.extend(template.environment.clone()); + } + environment.extend(spec.environment.clone()); + } + environment.retain(|key, _| !key.starts_with("OPENSHELL_")); + environment +} + /// Returns `true` when userns remaps all UIDs, making host-owned bind mounts /// unreadable from inside the container. `auto` and `no-map` remap every UID; /// `keep-id` preserves the host user's UID; `host` uses the host namespace. @@ -3269,6 +3285,51 @@ mod tests { } } + #[test] + fn child_environment_preserves_precedence_and_strips_control_keys() { + let mut sandbox = plain_sandbox("sandbox", "agent"); + sandbox.spec = Some(DriverSandboxSpec { + template: Some(DriverSandboxTemplate { + environment: HashMap::from([ + ("TEMPLATE_ONLY".to_string(), "template".to_string()), + ("OVERRIDE".to_string(), "template".to_string()), + ]), + ..Default::default() + }), + environment: HashMap::from([ + ("REQUEST_ONLY".to_string(), "request".to_string()), + ("OVERRIDE".to_string(), "request".to_string()), + ("OPENSHELL_SANDBOX_TOKEN".to_string(), "spoofed".to_string()), + ]), + ..Default::default() + }); + let image_env = vec![ + "IMAGE_ONLY=image".to_string(), + "OVERRIDE=image".to_string(), + "OPENSHELL_ENDPOINT=spoofed".to_string(), + ]; + + let environment = podman_child_environment(&sandbox, &image_env); + + assert_eq!( + environment.get("IMAGE_ONLY").map(String::as_str), + Some("image") + ); + assert_eq!( + environment.get("TEMPLATE_ONLY").map(String::as_str), + Some("template") + ); + assert_eq!( + environment.get("REQUEST_ONLY").map(String::as_str), + Some("request") + ); + assert_eq!( + environment.get("OVERRIDE").map(String::as_str), + Some("request") + ); + assert!(!environment.keys().any(|key| key.starts_with("OPENSHELL_"))); + } + fn secret_delete_request(sandbox_id: &str) -> String { format!( "DELETE {}", @@ -3289,22 +3350,17 @@ mod tests { "sha256:image".into(), ) .unwrap(); - let authentication = launch_authentication(); - let bundle = crate::isolation::bootstrap_archives( - "sandbox-1", - "ctr-1", - &identity, - HashMap::new(), - &authentication, - ) - .unwrap() - .channel; + let bundle = serde_json::to_vec(&crate::isolation::RestartMetadata { + workload_identity: identity, + child_env: HashMap::new(), + }) + .unwrap(); let mut header = tar::Header::new_gnu(); header.set_size(bundle.len() as u64); header.set_mode(0o600); header.set_cksum(); archive - .append_data(&mut header, "sandbox-bundle.tar", bundle.as_slice()) + .append_data(&mut header, "restart-metadata.json", bundle.as_slice()) .unwrap(); vec![ StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor stop diff --git a/crates/openshell-driver-podman/src/isolation.rs b/crates/openshell-driver-podman/src/isolation.rs index 09c6b5339e..6ab0c5bbc8 100644 --- a/crates/openshell-driver-podman/src/isolation.rs +++ b/crates/openshell-driver-podman/src/isolation.rs @@ -4,6 +4,7 @@ //! Podman-owned provisioning for the common authenticated isolation channel. use std::collections::{BTreeMap, HashMap}; +#[cfg(test)] use std::io::Read; use std::path::PathBuf; @@ -15,6 +16,7 @@ use openshell_sandbox_backend::boundary_protocol::{ SandboxTlsClientConfig, SandboxTlsServerConfig, SandboxTransport, generate_sandbox_tls_material, }; +use serde::{Deserialize, Serialize}; pub const LABEL_ROLE: &str = "openshell.io/isolation-role"; pub const WORKLOAD_FILTER: &str = "openshell.io/isolation-role=sandbox"; @@ -22,7 +24,7 @@ pub const CHANNEL_ROOT: &str = "/.openshell/channel"; pub const BOOTSTRAP_PATH: &str = "/.openshell/channel/sandbox/bootstrap.json"; pub const RUNTIME_DESCRIPTOR_PATH: &str = "/.openshell/supervisor/runtime-descriptor.json"; pub const AUTH_BUNDLE_PATH: &str = "/.openshell/supervisor/auth.json"; -pub const RESTART_BUNDLE_PATH: &str = "/.openshell/supervisor/sandbox-bundle.tar"; +pub const RESTART_METADATA_PATH: &str = "/.openshell/supervisor/restart-metadata.json"; const SOCKET_PATH: &str = "/.openshell/channel/sandbox/control.sock"; pub fn supervisor_name(id: &str) -> String { @@ -130,6 +132,12 @@ pub struct BootstrapArchives { pub supervisor: Vec, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RestartMetadata { + pub(crate) workload_identity: ResolvedWorkloadIdentity, + pub(crate) child_env: HashMap, +} + /// The shared volume contains only sandbox credentials. Supervisor credentials, /// gateway authorization, and the restart copy never enter that volume. pub fn bootstrap_archives( @@ -184,7 +192,7 @@ pub fn bootstrap_archives( resource_claim_files: BTreeMap::new(), workload_identity: identity.clone(), driver_fence: driver_fence.clone(), - child_env, + child_env: child_env.clone(), }; let runtime_descriptor = SandboxRuntimeDescriptor { boundary_id: sandbox_id.into(), @@ -228,7 +236,14 @@ pub fn bootstrap_archives( AUTH_BUNDLE_PATH, &serde_json::to_vec(&launch_authentication.supervisor).map_err(invalid)?, )?; - supervisor.file(RESTART_BUNDLE_PATH, &channel)?; + let restart_metadata = RestartMetadata { + workload_identity: identity.clone(), + child_env, + }; + supervisor.file( + RESTART_METADATA_PATH, + &serde_json::to_vec(&restart_metadata).map_err(invalid)?, + )?; Ok(BootstrapArchives { channel, workspace: workspace.finish()?, @@ -236,22 +251,8 @@ pub fn bootstrap_archives( }) } -pub fn boundary_config_from_channel_archive( - archive: &[u8], -) -> Result { - for entry in tar::Archive::new(archive).entries().map_err(invalid)? { - let mut entry = entry.map_err(invalid)?; - if entry.path().map_err(invalid)?.as_ref() != std::path::Path::new("sandbox/bootstrap.json") - { - continue; - } - let mut bytes = Vec::new(); - entry.read_to_end(&mut bytes).map_err(invalid)?; - return serde_json::from_slice(&bytes).map_err(invalid); - } - Err(ComputeDriverError::Precondition( - "Podman restart bundle has no sandbox bootstrap".to_string(), - )) +pub fn restart_metadata_from_slice(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes).map_err(invalid) } struct Archive<'a> { @@ -381,11 +382,12 @@ mod tests { ) .unwrap(); let authentication = authentication(); + let child_env = HashMap::from([("PATH".to_string(), "/agent/bin".to_string())]); let archives = bootstrap_archives( "sandbox", "container", &identity, - HashMap::new(), + child_env.clone(), &authentication, ) .unwrap(); @@ -427,11 +429,21 @@ mod tests { assert_eq!(config.driver_fence, runtime_descriptor.driver_fence); assert_eq!(config.workload_identity, identity); runtime_descriptor.driver_fence.validate().unwrap(); - assert_eq!( + let restart_metadata: RestartMetadata = serde_json::from_slice( supervisor - .get(&PathBuf::from(RESTART_BUNDLE_PATH.trim_start_matches('/'))) + .get(&PathBuf::from( + RESTART_METADATA_PATH.trim_start_matches('/'), + )) .unwrap(), - &archives.channel + ) + .unwrap(); + assert_eq!(restart_metadata.workload_identity, identity); + assert_eq!(restart_metadata.child_env, child_env); + let restart_bytes = serde_json::to_vec(&restart_metadata).unwrap(); + assert!( + !restart_bytes + .windows(b"PRIVATE KEY".len()) + .any(|window| window == b"PRIVATE KEY") ); } } diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index e60d576f97..7d2f5bb790 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -22,26 +22,28 @@ Gateway CLI flag > gateway OPENSHELL_* env var > TOML file > built-in defa ## Package-Managed Locations -Package-managed gateways do not require a TOML file. Create one at the package's optional config location when you need to override built-in defaults. Set `OPENSHELL_GATEWAY_CONFIG` in the launch environment to use a different file. +Package-managed gateways use either built-in defaults or a package-seeded TOML file. Set `OPENSHELL_GATEWAY_CONFIG` in the launch environment to use a different file. -| Package | Optional Gateway TOML location | +| Package | Gateway TOML location | |---|---| | Homebrew | `$XDG_CONFIG_HOME/openshell/gateway.toml` when it exists, otherwise the Homebrew prefix config such as `/opt/homebrew/var/openshell/gateway.toml`. | | Debian/Ubuntu | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml` for the systemd user service. | -| Fedora/RHEL RPM | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml` for the systemd user service. | +| Fedora/RHEL RPM | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml`; the systemd user service seeds this file from the packaged template on first start. | | Snap | `$SNAP_COMMON/gateway.toml`, usually `/var/snap/openshell/common/gateway.toml`. | The Fedora/RHEL RPM template leaves `[openshell.gateway].bind_address` unset. The gateway therefore uses its built-in `127.0.0.1:17670` primary listener. The Podman driver negotiates separate, restricted listeners for sandbox callbacks, so the primary listener does not need a wildcard address. Set `bind_address` explicitly only when clients must reach the primary multiplexed API through another interface. -The Homebrew formula creates its prefix config without setting `bind_address`, so the gateway uses its built-in `127.0.0.1:17670` primary listener. Docker Desktop and Podman Machine reuse that listener for sandbox callbacks. A user config takes precedence. Upgrades preserve user-edited configs and migrate only an unchanged prefix config generated with the affected IPv6-loopback default. +The Homebrew formula creates its prefix config without setting `bind_address`, so the gateway uses its built-in `127.0.0.1:17670` primary listener. Docker Desktop and Podman Machine reuse that listener for sandbox callbacks. A user config takes precedence. + +Homebrew and RPM upgrades migrate only exact package-generated schema-v1 defaults. Homebrew recognizes both its empty v1 prefix config and the affected IPv6-loopback variant. RPM recognizes the v1 file seeded by its systemd user service. Package upgrades never rewrite an edited file; migrate an edited v1 file manually with the steps below. ## Layout -The file is rooted at `[openshell]`. Gateway-wide settings live under `[openshell.gateway]`. Each compute driver owns its own `[openshell.drivers.]` table. Credential drivers own `[openshell.credential_drivers.]` tables. Shared compute-driver keys set at gateway scope are inherited into compute driver tables when not overridden. +The file is rooted at `[openshell]`. Gateway-wide settings live under `[openshell.gateway]`. Each compute driver owns its own `[openshell.drivers.]` table. Credential drivers own `[openshell.credential_drivers.]` tables. Driver-specific values are never inherited from gateway scope. ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] # ... gateway-wide settings ... @@ -114,7 +116,7 @@ A complete gateway configuration covering every section. Trim to the fields you # SPDX-License-Identifier: Apache-2.0 [openshell] -version = 1 +version = 2 [openshell.gateway] name = "production-us-west" @@ -124,15 +126,14 @@ metrics_bind_address = "0.0.0.0:9090" log_level = "info" -# When empty, the gateway auto-detects Kubernetes, then Podman, then Docker. +# When omitted, the gateway auto-detects Kubernetes, then Podman, then Docker. # VM is never auto-detected and requires an explicit entry here. -compute_drivers = ["kubernetes"] +compute_driver = "kubernetes" # Optional external provider credential storage backend. Omit this key to use # the gateway's default encrypted database credential storage. credential_drivers = ["kubernetes-secrets"] -sandbox_namespace = "openshell" ssh_session_ttl_secs = 3600 # Reject invalid policy generations securely by default. Set @@ -149,18 +150,11 @@ enable_loopback_service_http = true # Set true only for local plaintext gateways or trusted TLS termination. disable_tls = false -# Shared driver defaults. These inherit into [openshell.drivers.] tables -# when the driver-specific table does not override them. -default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -# Defaults to the gateway version; override to pin a specific build. -sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:" -# Defaults to the gateway version; override to pin a specific build. -# supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -client_tls_secret_name = "openshell-client-tls" -service_account_name = "openshell-sandbox" -host_gateway_ip = "10.0.0.1" -enable_user_namespaces = false -sa_token_ttl_secs = 3600 +# Guest TLS paths remain gateway settings. TLS-enabled Docker, Podman, and VM +# gateways require a complete bundle unless package-managed local TLS supplies +# it automatically. Omit all three when TLS is disabled. Kubernetes projects +# sandbox TLS from client_tls_secret_name instead. Driver tables must not repeat +# these fields. guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" guest_tls_key = "/etc/openshell/certs/client-key.pem" @@ -194,7 +188,6 @@ timeout = "500ms" cert_path = "/etc/openshell/certs/gateway.pem" key_path = "/etc/openshell/certs/gateway-key.pem" client_ca_path = "/etc/openshell/certs/client-ca.pem" -require_client_auth = false # Optional: SNI-based dual certificate for external (e.g. ACME) TLS. # external_cert_path = "/etc/openshell/certs/external.pem" # external_key_path = "/etc/openshell/certs/external-key.pem" @@ -205,7 +198,7 @@ signing_key_path = "/etc/openshell/jwt/signing.pem" public_key_path = "/etc/openshell/jwt/public.pem" kid_path = "/etc/openshell/jwt/kid" gateway_id = "openshell" -# Omit or set to 0 only for local single-player Docker, Podman, or VM gateways. +# Omit only for local single-player Docker, Podman, or VM gateways. ttl_secs = 3600 [openshell.gateway.auth] @@ -248,18 +241,33 @@ failure_policy = "fail_closed" rpc = "openshell.v1.OpenShell/UpdateConfig" phases = ["validate"] +[openshell.drivers.kubernetes] +namespace = "openshell" +# Required in raw TOML; Helm derives this from the gateway Service. +grpc_endpoint = "https://openshell-gateway.openshell.svc:8080" +default_image = "ghcr.io/nvidia/openshell/sandbox:latest" +# Defaults to the gateway version; override to pin a specific build. +# supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" +client_tls_secret_name = "openshell-client-tls" +service_account_name = "openshell-sandbox" +host_gateway_ip = "10.0.0.1" +enable_user_namespaces = false +sa_token_ttl_secs = 3600 + [openshell.credential_drivers.kubernetes-secrets] namespace = "openshell" allow_reference_namespace = false ``` -Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth] enabled = true` to authenticate CLI callers from verified client certificates. Kubernetes deployments must leave this unset and use OIDC or a trusted access proxy; the Helm chart does not render this table. +Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth] enabled = true` to map a verified client certificate to a CLI user identity. This application-layer identity switch does not control the TLS handshake. When `client_ca_path` is set without OIDC, the listener requires a valid client certificate. When OIDC is configured, bearer-only clients may connect; the listener still validates any client certificate they present against the configured CA. Kubernetes deployments must leave `mtls_auth.enabled` unset and use OIDC or a trusted access proxy; the Helm chart does not render this table. + +The client-certificate handshake policy is derived and has no `require_client_auth` TOML field. This preserves bearer-only OIDC clients and prevents a file setting from silently weakening CA-only gateways. `[openshell.gateway.tls]` supports optional SNI-based dual-certificate mode for deployments that need separate internal and external server certificates. Set `external_cert_path` and `external_key_path` to point at the external (e.g. ACME/publicly-trusted) certificate and key. List the hostnames that should be served with the external certificate in `external_server_names`. Connections whose TLS SNI hostname matches one of those names receive the external certificate; all other connections (including those with no SNI) receive the primary internal certificate from `cert_path`/`key_path`. Both fields must be set together — providing only one is a configuration error. On Kubernetes with the Helm chart, the external certificate is managed automatically when `certManager.serverIssuerRef.name` is set; the chart populates these fields from the cert-manager-issued external server certificate. `[openshell.gateway] policy_validation_failure_mode` controls what sandbox supervisors do when a complete candidate policy fails runtime validation. The default, `fail_closed`, deactivates the previous network policy, closes relays pinned to it, and denies new egress until a valid generation loads. `retain_last_valid` leaves the previous valid generation active. Both modes reject the candidate atomically; startup always fails closed when no previous valid generation exists. Gateway mutation paths that can preflight a known effective scope reject invalid candidates before persistence and leave the active policy unchanged regardless of this setting. Changing the value requires restarting the gateway so it can reload `gateway.toml` and distribute the new posture to sandbox supervisors. -`[openshell.gateway.gateway_jwt] ttl_secs` controls gateway-minted sandbox JWT lifetime. It defaults to `3600` seconds and must be between `60` and `3600` seconds so sandbox sessions can rotate short-lived credentials safely. +`[openshell.gateway.gateway_jwt] ttl_secs` controls gateway-minted sandbox JWT lifetime. Omit it for a non-expiring token: the token `exp` claim and `expires_at_ms` response field become `0`. Use this only for local single-player Docker, Podman, or VM gateways. Explicit `0` is invalid. Kubernetes and other shared deployments should set a positive TTL; Helm renders `3600` seconds by default, and the gateway logs a warning when a Kubernetes gateway omits the field. `[openshell.gateway.auth] allow_unauthenticated_users = true` is an unsafe local-development and trusted-proxy escape hatch. It accepts user-facing CLI/API calls without OIDC or mTLS credentials while sandbox supervisors still authenticate with gateway-minted sandbox JWTs. Leave it false for shared and production gateways. @@ -403,7 +411,7 @@ The gateway validates snapshot structure and provider-profile semantics. It trea `failure_policy` accepts `fail_closed` or `fail_open`. `timeout` accepts `ms` and `s` suffixes. In `dynamic` mode, binding overrides may select a manifest binding by `id`, `rpc`, or `service` plus `method`; they can disable a binding, narrow its phases, or override its failure policy. -`image_pull_policy` is intentionally not a shared gateway key. Kubernetes and Docker use `Always`, `IfNotPresent`, or `Never`. Podman uses `always`, `missing`, `never`, or `newer`. Set it inside the relevant driver table. +`image_pull_policy` is a shared driver setting with the canonical values `always`, `if_not_present`, `never`, and `newer`. Set it inside the relevant driver table. Drivers translate these values to their runtime APIs; `newer` is supported only by Podman and is rejected at Docker and Kubernetes startup. ## Credential Drivers @@ -492,7 +500,9 @@ args = [ ## Driver References -Each example is a complete TOML file for one compute driver. The examples repeat `[openshell]` and `[openshell.gateway]` so they stay copyable, and the driver tables list the accepted driver-specific keys. Driver-specific values override inherited gateway defaults. The gateway rejects unknown driver fields after inheritance is merged. +Each example is a complete TOML file for one compute driver. The examples repeat `[openshell]` and `[openshell.gateway]` so they stay copyable, and the driver tables list the accepted driver-specific keys. Drivers receive only their own tables, and the gateway rejects unknown gateway and driver fields. + +Kubernetes configurations set `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`. Docker configurations use `sandbox_label`; the legacy `sandbox_namespace` key is rejected. ### Kubernetes @@ -500,14 +510,14 @@ The gateway runs as a Pod and creates sandbox Pods in another namespace. mTLS ma ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "0.0.0.0:8080" health_bind_address = "0.0.0.0:8081" metrics_bind_address = "0.0.0.0:9090" log_level = "info" -compute_drivers = ["kubernetes"] +compute_driver = "kubernetes" [openshell.gateway.tls] cert_path = "/etc/openshell-tls/server/tls.crt" @@ -530,15 +540,14 @@ workspace_mode = "shared" namespace = "agents" service_account_name = "openshell-sandbox" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "IfNotPresent" +image_pull_policy = "if_not_present" image_pull_secrets = ["regcred"] -# Statically linked musl workload-side runtime. +# Defaults to the gateway version; override to pin a specific build. # sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:" -sandbox_runtime_image_pull_policy = "IfNotPresent" -# Dynamically linked glibc control-side runtime. +sandbox_runtime_image_pull_policy = "if_not_present" # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -supervisor_image_pull_policy = "IfNotPresent" +supervisor_image_pull_policy = "if_not_present" # Optional corporate HTTP forward proxy for policy-approved TLS egress. The # sandbox workload cannot select or override these settings. Only http:// proxy @@ -564,7 +573,10 @@ supervisor_image_pull_policy = "IfNotPresent" # Last resort for hostname-filtering proxy ACLs. The proxy resolves the target, # so its ACL becomes part of the egress boundary for proxied connections. # proxy_connect_by_hostname = true -grpc_endpoint = "https://openshell-gateway.agents.svc:8080" +# Required in raw gateway TOML because `namespace` identifies sandbox +# placement, not the gateway Service. Helm renders this from the release's +# gateway Service name and namespace. +grpc_endpoint = "https://openshell-gateway.openshell.svc:8080" ssh_socket_path = "/run/openshell/ssh.sock" client_tls_secret_name = "openshell-client-tls" host_gateway_ip = "10.0.0.1" @@ -624,18 +636,56 @@ The gateway verifies supervisor JWT-SVIDs with JWT bundles fetched from the SPIFFE Workload API, so this validation path does not require gateway access to the SPIRE OIDC discovery endpoint or its TLS CA. +### MXC + +The MXC driver runs Windows workloads through `wxc-exec`. Enable ETW auditing to +map Windows Sandboxing provider events into the gateway's OCSF stream. + +```toml +[openshell] +version = 2 + +[openshell.gateway] +bind_address = "127.0.0.1:17670" +log_level = "info" +compute_driver = "mxc" + +[openshell.drivers.mxc] +wxc_exec_path = "C:\\mxc-kit\\bin\\wxc-exec.exe" +backend = "process_container" +default_configuration_id = "composable" +pc_least_privilege = false +pc_capabilities = [] +debug = false +etw_audit = true +``` + +`etw_audit` defaults to `false`. When enabled, the gateway account must be an +administrator or belong to the Windows Performance Log Users group. Workload +commands and working directories remain sandbox-scoped and must be supplied in +the `mxc` driver configuration when creating a sandbox. + +The driver records executable identity in process audit events and omits raw +command arguments because they can contain credentials or personal data. See +[OCSF JSON Export](/observability/ocsf-json-export) for durable Windows audit +output. + ### Docker -Sandboxes run as containers on a local bridge network. The supervisor binary is bind-mounted from the host (no in-cluster image pull required); guest mTLS material is supplied as host paths. +Sandboxes run as containers on a local bridge network. The supervisor binary is bind-mounted from the host (no in-cluster image pull required). Configure guest mTLS paths once under `[openshell.gateway]`; the gateway validates and injects the bundle into the selected local driver. ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" -compute_drivers = ["docker"] +compute_driver = "docker" +# Gateway-owned bundle injected into the selected local driver. +guest_tls_ca = "/etc/openshell/certs/ca.pem" +guest_tls_cert = "/etc/openshell/certs/client.pem" +guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.docker] socket_path = "/var/run/docker.sock" @@ -647,25 +697,21 @@ sandbox_label = "docker-dev" # Optional override. When omitted, the gateway derives # https://host.openshell.internal: for this driver. grpc_endpoint = "https://host.openshell.internal:17670" -# Statically linked musl workload-side runtime. Defaults to the gateway version. +# Workload-side runtime. Defaults to the gateway version. # sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:" -# Dynamically linked glibc control-side runtime. Defaults to the gateway version. +# Supervisor runtime. Defaults to the gateway version. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" network_name = "openshell-docker" host_gateway_ip = "172.17.0.1" # Unsafe operator override. Host bind mounts, including Docker local-driver # bind-backed volumes, expose gateway-host paths inside sandboxes and can # negate OpenShell isolation and filesystem controls. enable_bind_mounts = false -# Set to 0 to leave Docker's runtime default unchanged. +# Omit to use OpenShell's 2048-process default. Explicit 0 is invalid. sandbox_pids_limit = 2048 -# Omit this field to keep Docker's runtime-selected AppArmor profile. -# Localhost/ requires an operator-loaded profile; Unconfined is an -# explicit operator opt-out. -# app_armor_profile = "Localhost/openshell-sandbox" +# Explicit supervisor-compatible default. RuntimeDefault requires Docker to +# report AppArmor support; Localhost/ requires an operator-loaded profile. +app_armor_profile = "Unconfined" # Corporate TLS egress proxy. These are supervisor argv settings, not workload # environment variables. Do not embed credentials in the URL. https_proxy = "https://proxy.corp.example:8443" @@ -678,18 +724,25 @@ proxy_auth_file = "/etc/openshell/secrets/proxy-auth" provider_spiffe_workload_api_socket = "/run/spire/agent.sock" ``` +Use `sandbox_label` for Docker configurations. The legacy +`sandbox_namespace` key is rejected. + ### Podman -Each Podman sandbox has a workload container running `openshell-sandbox` with `network=none`, and a separate `openshell-supervisor` companion on the configured network. Both use non-root identities, drop all capabilities, and keep the runtime's default seccomp profile. A private named volume carries their authenticated gRPC Unix socket. Gateway JWTs, optional gateway mTLS material, and upstream proxy credentials are delivered only to the supervisor; user mounts and GPU devices stay with the workload. +Each Podman sandbox uses two containers. The workload container runs `openshell-sandbox` with `network=none`; the supervisor container runs on the host network and initiates policy-approved upstream connections. A private volume carries their authenticated Unix-domain socket. Configure guest mTLS paths once under `[openshell.gateway]`; the gateway validates and injects the bundle into the selected local driver. ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" -compute_drivers = ["podman"] +compute_driver = "podman" +# Gateway-owned bundle injected into the selected local driver. +guest_tls_ca = "/etc/openshell/certs/ca.pem" +guest_tls_cert = "/etc/openshell/certs/client.pem" +guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.podman] # Rootless socket path. For root Podman use /run/podman/podman.sock. @@ -698,7 +751,8 @@ compute_drivers = ["podman"] # one. Set this to pin a specific Podman machine instead. socket_path = "/run/user/1000/podman/podman.sock" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "missing" # always | missing | never | newer +image_pull_policy = "if_not_present" # always | if_not_present | never | newer +# Optional override. When omitted, the gateway derives this endpoint. grpc_endpoint = "https://host.containers.internal:17670" # The gateway overwrites gateway_port from bind_address at runtime. gateway_port = 17670 @@ -706,25 +760,21 @@ network_name = "openshell" # Omit for the platform default: empty on Linux, 192.168.127.254 on macOS Podman machine. # Set "" to force Podman's host-gateway resolver. # host_gateway_ip = "192.168.127.254" -sandbox_ssh_socket_path = "/run/openshell/ssh.sock" +ssh_socket_path = "/run/openshell/ssh.sock" stop_timeout_secs = 45 -# Statically linked musl workload-side runtime. Defaults to the gateway version. +# Statically linked workload-side runtime. Defaults to the gateway version. # sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:" -# Dynamically linked glibc control-side runtime. Defaults to the gateway version. +# Dynamically linked supervisor runtime. Defaults to the gateway version. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" # Unsafe operator override. Host bind mounts, including Podman local-driver # bind-backed volumes, expose gateway-host paths inside sandboxes and can # negate OpenShell isolation and filesystem controls. enable_bind_mounts = false -# Set to 0 to leave Podman's runtime default unchanged. +# Omit to use OpenShell's 2048-process default. Explicit 0 is invalid. sandbox_pids_limit = 2048 -# Health check interval in seconds. Lower values detect readiness faster -# but increase process churn (each check spawns a conmon subprocess). -# Set to 0 to use a one-second check. Readiness checks cannot be disabled. -# Default: 10. +# Health check interval in seconds. Omit to disable health checks; explicit 0 +# is invalid. Lower values detect readiness faster but increase process churn +# (each check spawns a conmon subprocess). health_check_interval_secs = 10 # User namespace mode for sandbox containers. Omit to use the default. # Supported modes: auto, host, keep-id, no-map, private. @@ -735,7 +785,7 @@ health_check_interval_secs = 10 # rootful Podman uses absolute host IDs (0:1000:1, 1:100000:65536). # uidmap = ["0:0:1", "1:1:65535"] # gidmap = ["0:0:1", "1:1:65535"] -# Corporate forward proxy for sandbox egress. When set, the external +# Corporate forward proxy for sandbox egress. When set, the in-container # supervisor chains policy-approved TLS tunnels through this proxy with HTTP # CONNECT instead of dialing destinations directly. Plain-HTTP requests are # not proxied and always dial the destination directly. http:// and https:// @@ -813,21 +863,39 @@ health_check_interval_secs = 10 # proxy_connect_by_hostname = true # Corporate CA trusted for an https:// proxy and TLS-intercepting proxies. # proxy_ca_bundle = "/etc/openshell/tls/proxy-ca.pem" +# Project a host Workload API Unix socket into the supervisor, or use an +# explicit container-reachable TCP endpoint, for provider token exchange. +# provider_spiffe_workload_api_socket = "/run/spire/agent.sock" +# provider_spiffe_workload_api_socket = "tcp:169.254.1.2:8081" +# Omit app_armor_profile to preserve Podman's runtime-selected workload profile. +# Explicit RuntimeDefault and Localhost/ require Podman to report +# AppArmor support. +# app_armor_profile = "RuntimeDefault" ``` +Use `ssh_socket_path` for Podman configurations. The legacy +`sandbox_ssh_socket_path` key is rejected. When `app_armor_profile` is omitted, +OpenShell sends no override and Podman applies its runtime-selected profile. +The setting applies to the workload container; the supervisor retains Podman's +runtime-selected profile. + ### MicroVM Each sandbox runs inside its own libkrun microVM managed by the standalone `openshell-driver-vm` subprocess. Use this driver when you want stronger isolation than container namespaces alone. ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" # VM is never auto-detected; an explicit entry here is required. -compute_drivers = ["vm"] +compute_driver = "vm" +# Gateway-owned bundle injected into the selected local driver. +guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" +guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" +guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" [openshell.drivers.vm] state_dir = "/var/lib/openshell/vm" @@ -843,21 +911,20 @@ krun_log_level = 1 vcpus = 2 mem_mib = 2048 overlay_disk_mib = 4096 -guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" -guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" -guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" -# Resolved sandbox UID/GID for the rootfs /etc/passwd entry. -# Defaults to 10001 when unset; matching GID is used if sandbox_gid is empty. -# Any non-root Linux UID/GID is valid. +# Resolved sandbox UID/GID for new rootfs /etc/passwd entries. +# Defaults to the image's sandbox account, or 1000 when the account is absent; +# matching GID is used if sandbox_gid is empty. Persisted overlays recover their +# recorded identity, including 10001, rather than receiving a legacy fallback. +# Values must fall within OpenShell's allowed non-root sandbox identity range. # sandbox_uid = 20001 +# sandbox_gid = 20001 # Corporate forward proxy for sandbox egress. The keys, their semantics, and # the fail-closed contract are identical to the Podman driver above: only TLS # (CONNECT) egress is chained, plain-HTTP destination requests always dial # directly, credentials must come from proxy_auth_file rather than the URL, # an http:// proxy with credentials requires proxy_auth_allow_insecure, and # any present-but-invalid value is rejected at gateway startup rather than -# degrading to a direct dial. proxy_auth_file and proxy_ca_bundle are paths on -# the gateway host. +# degrading to a direct dial. proxy_auth_file is a path on the gateway host. # # The sandbox cannot select or override these settings. The driver passes them # only to the host supervisor. @@ -873,11 +940,18 @@ guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" # https_proxy = "http://host.openshell.internal:8080" # no_proxy = "10.0.0.0/8,.internal.example" # proxy_auth_file = "/etc/openshell/secrets/proxy-auth" +# An http:// proxy with proxy_auth_file requires this explicit acknowledgement: # proxy_auth_allow_insecure = true # Last resort for hostname-filtering proxy ACLs; see the Podman section above. # proxy_connect_by_hostname = true -# Corporate CA trusted for an https:// proxy and TLS-intercepting proxies. +# Gateway-host PEM bundle trusted for an https:// proxy and for server +# certificates re-signed by a TLS-intercepting proxy. Requires https_proxy. # proxy_ca_bundle = "/etc/openshell/tls/proxy-ca.pem" +# VM guests cannot mount a host Workload API Unix socket. Configure only a +# separately operated guest-reachable TCP listener and explicitly acknowledge +# the exposure; host-only sockets are never exposed automatically. +# provider_spiffe_workload_api_tcp_endpoint = "tcp:192.0.2.10:8081" +# provider_spiffe_allow_guest_tcp = true # Where the gateway stages rootfs tar archives for `--from ./rootfs.tar`. # Defaults to /rootfs-tar-staging. The gateway creates one # request-scoped subdirectory per staging slot and removes it after use. @@ -905,13 +979,65 @@ key used for driver-owned sandbox config such as `template.driver_config.` ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" -compute_drivers = ["kyma"] +compute_driver = "kyma" [openshell.drivers.kyma] socket_path = "/run/openshell/kyma-compute-driver.sock" ``` + +## Preflight package configuration {#gateway-config-preflight} + +Before starting a package-managed gateway, validate the selected file without +changing it: + +```shell +openshell-gateway config preflight --path ~/.config/openshell/gateway.toml +``` + +Without `--path`, the command validates a nonempty `OPENSHELL_GATEWAY_CONFIG`. +Otherwise, it validates an existing XDG gateway config when one is discovered. +When neither source selects a config, preflight succeeds. An explicit missing path, +a legacy schema-v1 file, invalid TOML, a symlink, or any nonregular file fails. +Preflight merges the selected file with the current `OPENSHELL_*` environment and +applies the daemon's read-only startup checks. These checks include selector and +socket normalization, registered-driver selection and configuration, rate-limit +pairs, TLS and mTLS relationships, interceptor registrations, and supervisor +middleware registrations. When a selected file omits `compute_driver`, preflight +validates each configured table for an auto-detectable driver without running the +runtime detection probes, which can connect local sockets or launch discovery +commands. It validates complete guest TLS path sets without requiring +package-generated certificates to exist before certificate generation. It does +not construct a compute driver or connect to a transport. A failed +preflight always preserves the file; it never migrates, replaces, or rewrites +configuration. + +To validate the exact daemon arguments that a wrapper will pass, place them after +`--` instead of using `--path`: + +```shell +openshell-gateway config preflight -- --config /etc/openshell/gateway.toml --grpc-rate-limit-requests 100 --grpc-rate-limit-window-seconds 60 +``` + +Debian and Ubuntu run preflight from the systemd user unit before local certificate +generation. The unit still loads the `gateway.env` environment file and starts the +gateway with no configuration arguments. Snap replays the exact effective daemon +arguments through preflight. It gives a nonempty `OPENSHELL_GATEWAY_CONFIG` +precedence; otherwise it validates and passes its canonical +`SNAP_COMMON/gateway.toml` only when that path exists in the filesystem. A broken +symlink is therefore rejected instead of being treated as absent. + +Package startup does not modify an operator-owned v1 file. Back it up, follow +[Migrate to schema version 2](#migrate-to-schema-version-2), then validate the +result explicitly before restarting the service: + +```shell +cp ~/.config/openshell/gateway.toml ~/.config/openshell/gateway.toml.v1.bak +$EDITOR ~/.config/openshell/gateway.toml +openshell-gateway config preflight --path ~/.config/openshell/gateway.toml +systemctl --user restart openshell-gateway +``` diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 691f7077ac..76548f60c7 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -42,11 +42,11 @@ with the exact exit code. Driver and supervisor failures remain `Error`. ## Configure a Compute Driver -Configure the compute driver on the gateway. Current releases accept one driver per gateway. Set `compute_drivers` in the gateway TOML file: +Configure the compute driver on the gateway. Current releases accept one driver per gateway. Set `compute_driver` in the gateway TOML file: ```toml [openshell.gateway] -compute_drivers = ["docker"] +compute_driver = "docker" ``` Reserved built-in values are `docker`, `podman`, `kubernetes`, `vm`, and `mxc`. @@ -54,13 +54,13 @@ The `mxc` driver is available only in native Windows gateway builds. Non-reserved names select an extension driver and require a `socket_path` in `[openshell.drivers.]`. -When `compute_drivers` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Docker must respond on a known API socket. Podman first probes known API sockets and then asks the `podman` CLI for the active native or machine-backed socket. The VM driver is never auto-detected; configure it explicitly with `compute_drivers = ["vm"]` or set `OPENSHELL_DRIVERS=vm` in the launch environment. +When `compute_driver` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Docker must respond on a known API socket. Podman first probes known API sockets and then asks the `podman` CLI for the active native or machine-backed socket. The VM driver is never auto-detected; configure it explicitly with `compute_driver = "vm"` or set `OPENSHELL_COMPUTE_DRIVER=vm` in the launch environment. Common gateway options: | Gateway TOML option | Description | |---|---| -| `compute_drivers = [""]` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | +| `compute_driver = ""` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | Set driver-specific values such as sandbox images, callback endpoints, network names, TLS material, and VM sizing in the gateway TOML file. See the [Gateway Configuration File](./gateway-config) reference for the full `[openshell.drivers.]` schema. @@ -70,7 +70,7 @@ the gateway at the Unix socket the operator has already provisioned: ```toml [openshell.gateway] -compute_drivers = ["kyma"] +compute_driver = "kyma" [openshell.drivers.kyma] socket_path = "/run/openshell/kyma.sock" @@ -175,7 +175,7 @@ that already covers loopback. Otherwise, the Docker driver requests a separate For maintainer-level implementation details, refer to the [Docker driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-docker/README.md). -Select Docker with `compute_drivers = ["docker"]` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_runtime_image`, `supervisor_image`, `image_pull_policy`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. The sandbox runtime image contains the static musl `/openshell-sandbox` binary; the supervisor image contains the dynamic glibc `/openshell-supervisor` binary. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. +Select Docker with `compute_driver = "docker"` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_runtime_image`, `supervisor_image`, `image_pull_policy`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. The sandbox runtime image contains `/openshell-sandbox`; the supervisor image contains `/openshell-supervisor`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. When operating `openshell-driver-docker` as an external driver, set `OPENSHELL_OTLP_ENDPOINT` to export its spans. The driver continues W3C trace @@ -255,7 +255,7 @@ The agent workload uses `network=none`. Its trusted supervisor companion uses Po For maintainer-level implementation details, refer to the [Podman driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/README.md) and [Podman networking notes](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/NETWORKING.md). -Select Podman with `compute_drivers = ["podman"]` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `sandbox_runtime_image`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `sandbox_ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. +Select Podman with `compute_driver = "podman"` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `sandbox_runtime_image`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. Podman sandboxes default to a 45-second graceful stop window before Podman escalates from `SIGTERM` to `SIGKILL`. Set `stop_timeout_secs` in gateway config, or `OPENSHELL_STOP_TIMEOUT` for the standalone driver, when a local runtime needs a different teardown window. @@ -349,14 +349,14 @@ For maintainer-level implementation details, refer to the [VM driver README](htt The VM driver is opt-in. Release packages can install `openshell-driver-vm`, but the gateway does not select it unless you configure the driver explicitly. -Enable VM by setting `compute_drivers = ["vm"]` in the gateway TOML file: +Enable VM by setting `compute_driver = "vm"` in the gateway TOML file: ```toml [openshell.gateway] -compute_drivers = ["vm"] +compute_driver = "vm" ``` -For a launch-time override, set `OPENSHELL_DRIVERS=vm` in the gateway environment and restart the service. +For a launch-time override, set `OPENSHELL_COMPUTE_DRIVER=vm` in the gateway environment and restart the service. Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, `krun_log_level`, and `guest_tls_*` in `[openshell.drivers.vm]`. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. The VM socket path is managed by the gateway and is not configurable through remote endpoint settings. @@ -414,7 +414,7 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | Gateway configuration | Helm value | Description | |---|---|---| -| `compute_drivers = ["kubernetes"]` | Not applicable | Select the Kubernetes compute driver. | +| `compute_driver = "kubernetes"` | Not applicable | Select the Kubernetes compute driver. | | `[openshell.drivers.kubernetes].namespace` | `server.sandboxNamespace` | Set the namespace for sandbox resources. The Helm chart defaults to the release namespace when left empty. | | `service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the Kubernetes driver's TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | | `default_image` | `server.sandboxImage` | Set the default sandbox image. | @@ -423,9 +423,9 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `[managed_ssh_ingress]` | `networkPolicy.enabled` | In managed mode, create an SSH ingress policy in every workspace namespace. Helm configures the gateway namespace and pod selector automatically. Operator mode leaves namespace policy management to the platform operator. | | `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. | | `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. | -| `sandbox_runtime_image` | `sandboxRuntime.image.repository` / `sandboxRuntime.image.tag` | Override the image that provides the static musl `openshell-sandbox` binary. The default repository with an empty tag uses the version pinned into the gateway. | +| `sandbox_runtime_image` | `sandboxRuntime.image.repository` / `sandboxRuntime.image.tag` | Override the image that provides `openshell-sandbox`. The default repository with an empty tag uses the version pinned into the gateway. | | `sandbox_runtime_image_pull_policy` | `sandboxRuntime.image.pullPolicy` | Set the Kubernetes image pull policy for the sandbox runtime image. | -| `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the image that provides the dynamic glibc `openshell-supervisor` binary. The default repository with an empty tag uses the version pinned into the gateway. | +| `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the image that provides `openshell-supervisor`. The default repository with an empty tag uses the version pinned into the gateway. | | `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | | `sandbox_runtime.network_policy_enforced` | `supervisor.sandboxRuntime.networkPolicyEnforced` | Acknowledge that the cluster CNI enforces ingress and egress `NetworkPolicy` in sandbox namespaces. This must be `true`. | | `sandbox_runtime.boundary_port` | `supervisor.sandboxRuntime.boundaryPort` | Set the non-privileged TLS port used between the paired supervisor and sandbox Pods. | From a2850d833e50b47e7b21e9641a7867989b454f98 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 13 Sep 2026 20:00:39 -0700 Subject: [PATCH 11/19] fix(podman): inspect Debian supervisor provenance Signed-off-by: Drew Newberry --- e2e/with-podman-gateway.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index f7f3d48401..5f09d239cb 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -578,6 +578,10 @@ if ! [[ "${SUPERVISOR_RUNTIME_IMAGE}" =~ ^[^@]+@sha256:[0-9a-f]{64}$ ]]; then exit 2 fi SUPERVISOR_BASE_IMAGE="$(awk '$1 == "FROM" { print $2; exit }' "${OPENSHELL_E2E_SUPERVISOR_DOCKERFILE:-${ROOT}/deploy/docker/Dockerfile.supervisor}")" +if ! podman_cmd image exists "${SUPERVISOR_BASE_IMAGE}" 2>/dev/null; then + echo "Pulling Podman supervisor base image ${SUPERVISOR_BASE_IMAGE}..." + podman_cmd pull "${SUPERVISOR_BASE_IMAGE}" +fi SUPERVISOR_BASE_IMAGE_ID="$(podman_cmd image inspect --format '{{.Id}}' "${SUPERVISOR_BASE_IMAGE}")" SUPERVISOR_BASE_IMAGE_ID="${SUPERVISOR_BASE_IMAGE_ID#sha256:}" SUPERVISOR_BASE_IMAGE_DIGEST="$(podman_cmd image inspect --format '{{.Digest}}' "${SUPERVISOR_BASE_IMAGE}")" @@ -588,8 +592,9 @@ if ! [[ "${SUPERVISOR_BASE_IMAGE_ID}" =~ ^[0-9a-f]{64}$ ]] \ fi SUPERVISOR_PACKAGE_MANIFEST="${OPENSHELL_PARITY_SUPERVISOR_PACKAGE_CAPTURE:-${WORKDIR}/supervisor.packages.txt}" mkdir -p "$(dirname "${SUPERVISOR_PACKAGE_MANIFEST}")" -podman_cmd run --rm --network none --entrypoint /sbin/apk \ - "${SUPERVISOR_RUNTIME_IMAGE}" info -v | LC_ALL=C sort >"${SUPERVISOR_PACKAGE_MANIFEST}" +podman_cmd run --rm --network none --entrypoint /usr/bin/dpkg-query \ + "${SUPERVISOR_RUNTIME_IMAGE}" -W '-f=${binary:Package}=${Version}\n' \ + | LC_ALL=C sort >"${SUPERVISOR_PACKAGE_MANIFEST}" SUPERVISOR_PACKAGE_MANIFEST_SHA256="$(sha256sum "${SUPERVISOR_PACKAGE_MANIFEST}" | cut -d' ' -f1)" echo "Using Podman supervisor image: ${SUPERVISOR_RUNTIME_IMAGE} (ID ${SUPERVISOR_IMAGE_ID}, digest ${SUPERVISOR_IMAGE_DIGEST}, base ${SUPERVISOR_BASE_IMAGE} ID ${SUPERVISOR_BASE_IMAGE_ID} digest ${SUPERVISOR_BASE_IMAGE_DIGEST}, packages ${SUPERVISOR_PACKAGE_MANIFEST_SHA256})" From 2dea15a87eb0b9a493a3742a1e4c08759f053e6b Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 13 Sep 2026 21:20:37 -0700 Subject: [PATCH 12/19] fix(podman): use libpod-compatible tmpfs options Signed-off-by: Drew Newberry --- .../openshell-driver-podman/src/container.rs | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index e7ea5951da..9d3ded6e90 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1468,14 +1468,15 @@ pub fn build_isolation_specs( workload.mounts.push(Mount { kind: "tmpfs".into(), source: "tmpfs".into(), - destination: openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_DIR.into(), + destination: openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_ROOT.into(), options: vec![ "rw".into(), + "noexec".into(), "nosuid".into(), "nodev".into(), - format!("uid={}", input.identity.uid), - format!("gid={}", input.identity.gid), - "mode=0755".into(), + // Libpod's OCI `mounts` API rejects tmpfs uid/gid options. The + // unprivileged runtime creates the owned material subdirectory. + "mode=0777".into(), "size=1m".into(), ], }); @@ -1536,11 +1537,10 @@ pub fn build_isolation_specs( destination: destination.into(), options: vec![ "rw".into(), + "noexec".into(), "nosuid".into(), "nodev".into(), - format!("uid={}", input.identity.uid), - format!("gid={}", input.identity.gid), - "mode=0700".into(), + "mode=0777".into(), "size=64m".into(), ], }); @@ -1755,19 +1755,13 @@ mod tests { .workload .mounts .iter() - .find(|mount| mount.destination == openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_DIR) + .find(|mount| { + mount.destination == openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_ROOT + }) .expect("workload supervisor CA mount"); assert_eq!(supervisor_ca_mount.kind, "tmpfs"); assert_eq!(supervisor_ca_mount.source, "tmpfs"); - for option in [ - "rw", - "nosuid", - "nodev", - "uid=1000", - "gid=1001", - "mode=0755", - "size=1m", - ] { + for option in ["rw", "noexec", "nosuid", "nodev", "mode=0777", "size=1m"] { assert!(supervisor_ca_mount.options.contains(&option.to_string())); } assert!( From 78924dff1dbeb187c6271502d35641748514f360 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 13 Sep 2026 21:36:10 -0700 Subject: [PATCH 13/19] fix(podman): bind verified sandbox runtime binary Signed-off-by: Drew Newberry --- crates/openshell-driver-podman/src/config.rs | 4 +- crates/openshell-driver-podman/src/driver.rs | 61 +++++++++++--------- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 8b4e7f240b..d0ca11669e 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -61,8 +61,8 @@ pub struct PodmanComputeConfig { /// Container stop timeout in seconds (SIGTERM → SIGKILL). pub stop_timeout_secs: u32, /// OCI image containing the statically linked `openshell-sandbox` binary. - /// Mounted read-only into sandbox containers at /opt/openshell/bin - /// using Podman's `type=image` mount. + /// The driver extracts the binary from this image into a verified host + /// cache and mounts it read-only into each sandbox container. pub sandbox_runtime_image: String, /// OCI image containing the dynamically linked `openshell-supervisor` binary. pub supervisor_image: String, diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index f96ce723dd..542eb15b6b 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -799,8 +799,9 @@ impl PodmanComputeDriver { let (image, immutable_image_id, image_user, image_env) = async { let phase_status = openshell_otel::ErrorStatusGuard::current(); let result = async { - // The sandbox runtime is shipped in a standalone OCI image and - // mounted into workload containers via Podman's type=image mount. + // The sandbox runtime is shipped in a standalone OCI image. + // The driver extracts and verifies its binary before bind-mounting + // it into the workload container. let sandbox_runtime_pull_policy = runtime_image_pull_policy(&self.config.sandbox_runtime_image); info!( @@ -987,18 +988,17 @@ impl PodmanComputeDriver { return Err(e); } }; - let supervisor_bin_path = if userns_needs_extraction(self.config.userns.as_deref()) - { + // Podman's image-volume support varies across libpod/runtime + // combinations. Always use the verified extraction cache so + // workload startup does not depend on type=image mounts. + let supervisor_bin_path = match extract_sandbox_bin(&self.client, &runtime_config).await { Ok(path) => Some(path), Err(e) => { cleanup_created().await; return Err(e); } - } - } else { - None - }; + }; let tls_secret_names = if self.config.tls_enabled() { let names = container::tls_secret_names(&sandbox.id); @@ -1774,6 +1774,9 @@ async fn extract_sandbox_bin( let cache_path = openshell_core::driver_utils::supervisor_cache_path("podman-sandbox", digest) .map_err(ComputeDriverError::Precondition)?; + // Unit tests use ordered Podman API stubs and intentionally exercise the + // extraction path on every create. Production reuses the immutable cache. + #[cfg(not(test))] if cache_path.is_file() { validate_linux_elf_binary(&cache_path).map_err(ComputeDriverError::Precondition)?; info!( @@ -1836,13 +1839,6 @@ async fn extract_binary_from_container( Ok(cache_path.to_path_buf()) } -fn userns_needs_extraction(userns: Option<&str>) -> bool { - userns.is_some_and(|mode| { - let base = mode.split(':').next().unwrap_or(mode); - !base.eq_ignore_ascii_case("host") - }) -} - fn podman_child_environment( sandbox: &DriverSandbox, image_env: &[String], @@ -3444,6 +3440,22 @@ mod tests { ) } + fn sandbox_binary_archive_response() -> StubResponse { + let mut archive = Vec::new(); + { + let mut builder = tar::Builder::new(&mut archive); + let payload = b"\x7fELF-test-sandbox"; + let mut header = tar::Header::new_gnu(); + header.set_path("openshell-sandbox").unwrap(); + header.set_size(payload.len() as u64); + header.set_mode(0o755); + header.set_cksum(); + builder.append(&header, payload.as_slice()).unwrap(); + builder.finish().unwrap(); + } + StubResponse::new(StatusCode::OK, archive) + } + fn create_setup_responses(proxy_secret: bool) -> Vec { let mut responses = vec![ StubResponse::new(StatusCode::OK, "{}"), // sandbox runtime pull @@ -3462,6 +3474,12 @@ mod tests { if proxy_secret { responses.push(StubResponse::new(StatusCode::CREATED, "{}")); } + responses.extend([ + image_response("sha256:sandbox-runtime"), + created_response("sandbox-runtime-extractor"), + sandbox_binary_archive_response(), + StubResponse::new(StatusCode::NO_CONTENT, ""), // remove extractor + ]); responses.push(StubResponse::new(StatusCode::CREATED, "{}")); // channel volume responses } @@ -3707,19 +3725,6 @@ mod tests { let _ = fs::remove_file(socket_path); } - #[test] - fn userns_needs_extraction_cases() { - assert!(!userns_needs_extraction(None)); - assert!(!userns_needs_extraction(Some("host"))); - assert!(!userns_needs_extraction(Some("Host"))); - assert!(userns_needs_extraction(Some("auto"))); - assert!(userns_needs_extraction(Some("auto:size=65536"))); - assert!(userns_needs_extraction(Some("keep-id"))); - assert!(userns_needs_extraction(Some("keep-id:uid=1000"))); - assert!(userns_needs_extraction(Some("no-map"))); - assert!(userns_needs_extraction(Some("private"))); - } - #[test] fn userns_remaps_uids_cases() { assert!(!userns_remaps_uids(None)); From c683aa42b280622582ffafabff77a09167b301b9 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 13 Sep 2026 21:56:17 -0700 Subject: [PATCH 14/19] fix(podman): provide external driver data directory Signed-off-by: Drew Newberry --- e2e/parity/test.sh | 1 + e2e/parity/verify-results.py | 15 +++++++++++- e2e/with-podman-gateway.sh | 23 +++++++++++-------- ...chema_v2_compute_boundary_verifier_test.py | 6 ++++- 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/e2e/parity/test.sh b/e2e/parity/test.sh index 2b8124ccd6..b06f76678a 100755 --- a/e2e/parity/test.sh +++ b/e2e/parity/test.sh @@ -213,6 +213,7 @@ if external: "external_driver_proxy": False, "external_driver_app_armor": False, "external_driver_environment": { + "XDG_DATA_HOME": f"/tmp/{variant}-driver-data", "OPENSHELL_COMPUTE_DRIVER_SOCKET": driver_socket, "OPENSHELL_PODMAN_SOCKET": podman_socket, "OPENSHELL_SANDBOX_IMAGE": sandbox_runtime, diff --git a/e2e/parity/verify-results.py b/e2e/parity/verify-results.py index 8f4942dda3..4e8a4619bb 100644 --- a/e2e/parity/verify-results.py +++ b/e2e/parity/verify-results.py @@ -262,6 +262,7 @@ def verify_variant( sandbox_id = launch.get("sandbox_image_id") sandbox_digest = launch.get("sandbox_image_digest") sandbox_runtime = launch.get("sandbox_runtime_image") + sandbox_boundary_image = launch.get("sandbox_boundary_image") sandbox_match = ( DIGEST_REFERENCE_RE.fullmatch(sandbox_runtime) if isinstance(sandbox_runtime, str) @@ -285,6 +286,10 @@ def verify_variant( sandbox_request == sandbox_runtime, f"{launch_path}: sandbox image request was not the resolved digest reference", ) + require( + isinstance(sandbox_boundary_image, str) and sandbox_boundary_image, + f"{launch_path}: sandbox boundary image is missing", + ) if not external: require( podman_config["default_image"] == sandbox_runtime @@ -369,6 +374,7 @@ def verify_variant( ) driver_environment = launch.get("external_driver_environment") expected_environment_keys = { + "XDG_DATA_HOME", "OPENSHELL_COMPUTE_DRIVER_SOCKET", "OPENSHELL_PODMAN_SOCKET", "OPENSHELL_SANDBOX_IMAGE", @@ -378,6 +384,7 @@ def verify_variant( "OPENSHELL_GATEWAY_PORT", "OPENSHELL_NETWORK_NAME", "OPENSHELL_STOP_TIMEOUT", + "OPENSHELL_SANDBOX_RUNTIME_IMAGE", "OPENSHELL_SUPERVISOR_IMAGE", "OPENSHELL_PODMAN_TLS_CA", "OPENSHELL_PODMAN_TLS_CERT", @@ -389,6 +396,10 @@ def verify_variant( and set(driver_environment) == expected_environment_keys, f"{launch_path}: external driver allowlisted environment is incomplete", ) + require( + Path(driver_environment["XDG_DATA_HOME"]).is_absolute(), + f"{launch_path}: external driver data directory is not absolute", + ) require( driver_environment["OPENSHELL_COMPUTE_DRIVER_SOCKET"] == podman_config["socket_path"], @@ -402,7 +413,7 @@ def verify_variant( f"{launch_path}: external driver Podman socket is not isolated", ) require( - driver_environment["OPENSHELL_SANDBOX_IMAGE"] == sandbox_runtime + driver_environment["OPENSHELL_SANDBOX_IMAGE"] == sandbox_request and driver_environment["OPENSHELL_SANDBOX_IMAGE_PULL_POLICY"] == expected_policy and driver_environment["OPENSHELL_HEALTH_CHECK_INTERVAL_SECS"] == 10 @@ -412,6 +423,8 @@ def verify_variant( and driver_environment["OPENSHELL_NETWORK_NAME"] and isinstance(driver_environment["OPENSHELL_STOP_TIMEOUT"], int) and driver_environment["OPENSHELL_STOP_TIMEOUT"] >= 0 + and driver_environment["OPENSHELL_SANDBOX_RUNTIME_IMAGE"] + == sandbox_boundary_image and driver_environment["OPENSHELL_SUPERVISOR_IMAGE"] == runtime_image and driver_environment["OPENSHELL_ENABLE_BIND_MOUNTS"] is True, f"{launch_path}: external driver allowlisted runtime inputs differ", diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 5f09d239cb..de34d6c10c 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -128,6 +128,8 @@ DRIVER_PID="" DRIVER_LOG="${OPENSHELL_PARITY_EXTERNAL_DRIVER_LOG_CAPTURE:-${WORKDIR}/podman-driver.log}" mkdir -p "$(dirname "${DRIVER_LOG}")" DRIVER_SOCKET="${WORKDIR}/compute-driver.sock" +DRIVER_DATA_HOME="${WORKDIR}/driver-data" +mkdir -p "${DRIVER_DATA_HOME}" E2E_NAMESPACE="" PODMAN_NETWORK_NAME="" PODMAN_NETWORK_MANAGED=0 @@ -598,9 +600,9 @@ podman_cmd run --rm --network none --entrypoint /usr/bin/dpkg-query \ SUPERVISOR_PACKAGE_MANIFEST_SHA256="$(sha256sum "${SUPERVISOR_PACKAGE_MANIFEST}" | cut -d' ' -f1)" echo "Using Podman supervisor image: ${SUPERVISOR_RUNTIME_IMAGE} (ID ${SUPERVISOR_IMAGE_ID}, digest ${SUPERVISOR_IMAGE_DIGEST}, base ${SUPERVISOR_BASE_IMAGE} ID ${SUPERVISOR_BASE_IMAGE_ID} digest ${SUPERVISOR_BASE_IMAGE_DIGEST}, packages ${SUPERVISOR_PACKAGE_MANIFEST_SHA256})" -SANDBOX_RUNTIME_IMAGE="$(resolve_podman_sandbox_runtime_image)" -ensure_podman_sandbox_runtime_image "${SANDBOX_RUNTIME_IMAGE}" -echo "Using Podman sandbox runtime image: ${SANDBOX_RUNTIME_IMAGE}" +SANDBOX_BOUNDARY_IMAGE="$(resolve_podman_sandbox_runtime_image)" +ensure_podman_sandbox_runtime_image "${SANDBOX_BOUNDARY_IMAGE}" +echo "Using Podman sandbox runtime image: ${SANDBOX_BOUNDARY_IMAGE}" DEFAULT_SANDBOX_IMAGE="ghcr.io/nvidia/openshell-community/sandboxes/base:latest" SANDBOX_IMAGE_REQUEST="${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-${OPENSHELL_SANDBOX_IMAGE:-${DEFAULT_SANDBOX_IMAGE}}}" @@ -726,17 +728,18 @@ if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then driver_tls_ca_sha256="$(sha256sum "${EXTERNAL_DRIVER_TLS_CA}" | cut -d' ' -f1)" driver_tls_cert_sha256="$(sha256sum "${EXTERNAL_DRIVER_TLS_CERT}" | cut -d' ' -f1)" driver_tls_key_sha256="$(sha256sum "${EXTERNAL_DRIVER_TLS_KEY}" | cut -d' ' -f1)" - external_driver_environment="$(printf '{\"OPENSHELL_COMPUTE_DRIVER_SOCKET\":\"%s\",\"OPENSHELL_PODMAN_SOCKET\":\"%s\",\"OPENSHELL_SANDBOX_IMAGE\":\"%s\",\"OPENSHELL_SANDBOX_IMAGE_PULL_POLICY\":\"%s\",\"OPENSHELL_HEALTH_CHECK_INTERVAL_SECS\":%s,\"OPENSHELL_GRPC_ENDPOINT\":\"%s\",\"OPENSHELL_GATEWAY_PORT\":%s,\"OPENSHELL_NETWORK_NAME\":\"%s\",\"OPENSHELL_STOP_TIMEOUT\":%s,\"OPENSHELL_SANDBOX_RUNTIME_IMAGE\":\"%s\",\"OPENSHELL_SUPERVISOR_IMAGE\":\"%s\",\"OPENSHELL_PODMAN_TLS_CA\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_PODMAN_TLS_CERT\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_PODMAN_TLS_KEY\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_ENABLE_BIND_MOUNTS\":%s}' \ + external_driver_environment="$(printf '{\"XDG_DATA_HOME\":\"%s\",\"OPENSHELL_COMPUTE_DRIVER_SOCKET\":\"%s\",\"OPENSHELL_PODMAN_SOCKET\":\"%s\",\"OPENSHELL_SANDBOX_IMAGE\":\"%s\",\"OPENSHELL_SANDBOX_IMAGE_PULL_POLICY\":\"%s\",\"OPENSHELL_HEALTH_CHECK_INTERVAL_SECS\":%s,\"OPENSHELL_GRPC_ENDPOINT\":\"%s\",\"OPENSHELL_GATEWAY_PORT\":%s,\"OPENSHELL_NETWORK_NAME\":\"%s\",\"OPENSHELL_STOP_TIMEOUT\":%s,\"OPENSHELL_SANDBOX_RUNTIME_IMAGE\":\"%s\",\"OPENSHELL_SUPERVISOR_IMAGE\":\"%s\",\"OPENSHELL_PODMAN_TLS_CA\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_PODMAN_TLS_CERT\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_PODMAN_TLS_KEY\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_ENABLE_BIND_MOUNTS\":%s}' \ + "${DRIVER_DATA_HOME}" \ "${DRIVER_SOCKET}" \ "${OPENSHELL_PODMAN_SOCKET:-}" \ - "${SANDBOX_RUNTIME_IMAGE}" \ + "${SANDBOX_IMAGE_REQUEST}" \ "${EXTERNAL_DRIVER_PULL_POLICY}" \ "${EXTERNAL_DRIVER_HEALTH_CHECK_INTERVAL_SECS}" \ "${EXTERNAL_DRIVER_CALLBACK_ENDPOINT}" \ "${HOST_PORT}" \ "${PODMAN_NETWORK_NAME}" \ "${PODMAN_STOP_TIMEOUT_SECS}" \ - "${SANDBOX_RUNTIME_IMAGE}" \ + "${SANDBOX_BOUNDARY_IMAGE}" \ "${SUPERVISOR_RUNTIME_IMAGE}" \ "${EXTERNAL_DRIVER_TLS_CA}" \ "${driver_tls_ca_sha256}" \ @@ -746,7 +749,7 @@ if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then "${driver_tls_key_sha256}" \ "${EXTERNAL_DRIVER_ENABLE_BIND_MOUNTS}")" fi - printf '{"schema_version":%s,"gateway_port":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s","supervisor_image":"%s","supervisor_image_id":"%s","supervisor_image_digest":"%s","supervisor_runtime_image":"%s","supervisor_base_image":"%s","supervisor_base_image_id":"%s","supervisor_base_image_digest":"%s","supervisor_base_runtime_image":"%s","supervisor_package_manifest_sha256":"%s","sandbox_image_request":"%s","sandbox_image_id":"%s","sandbox_image_digest":"%s","sandbox_runtime_image":"%s","sandbox_client_image_alias":"%s","sandbox_client_image_alias_id":"%s","gateway_sha256_before_execution":"%s","cli_sha256_before_execution":"%s","conformance_sha256_before_execution":"%s","external_driver_sha256_before_execution":"%s","supervisor_sha256_before_execution":"%s","supervisor_dockerfile_sha256_before_execution":"%s","cli_trace_wrapper_sha256_before_execution":"%s","external_driver_grpc_endpoint":%s,"external_driver_host_gateway_ip":%s,"external_driver_userns":%s,"external_driver_spiffe":%s,"external_driver_proxy":%s,"external_driver_app_armor":%s,"external_driver_environment":%s}\n' \ + printf '{"schema_version":%s,"gateway_port":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s","supervisor_image":"%s","supervisor_image_id":"%s","supervisor_image_digest":"%s","supervisor_runtime_image":"%s","supervisor_base_image":"%s","supervisor_base_image_id":"%s","supervisor_base_image_digest":"%s","supervisor_base_runtime_image":"%s","supervisor_package_manifest_sha256":"%s","sandbox_image_request":"%s","sandbox_image_id":"%s","sandbox_image_digest":"%s","sandbox_runtime_image":"%s","sandbox_boundary_image":"%s","sandbox_client_image_alias":"%s","sandbox_client_image_alias_id":"%s","gateway_sha256_before_execution":"%s","cli_sha256_before_execution":"%s","conformance_sha256_before_execution":"%s","external_driver_sha256_before_execution":"%s","supervisor_sha256_before_execution":"%s","supervisor_dockerfile_sha256_before_execution":"%s","cli_trace_wrapper_sha256_before_execution":"%s","external_driver_grpc_endpoint":%s,"external_driver_host_gateway_ip":%s,"external_driver_userns":%s,"external_driver_spiffe":%s,"external_driver_proxy":%s,"external_driver_app_armor":%s,"external_driver_environment":%s}\n' \ "${CONFIG_SCHEMA_VERSION}" \ "${HOST_PORT}" \ "$([ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ] && printf true || printf false)" \ @@ -765,6 +768,7 @@ if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then "${SANDBOX_IMAGE_ID}" \ "${SANDBOX_IMAGE_DIGEST}" \ "${SANDBOX_RUNTIME_IMAGE}" \ + "${SANDBOX_BOUNDARY_IMAGE}" \ "${SANDBOX_CLIENT_IMAGE_ALIAS}" \ "${SANDBOX_CLIENT_IMAGE_ALIAS_ID}" \ "${OPENSHELL_E2E_EXPECTED_GATEWAY_SHA256:-}" \ @@ -788,16 +792,17 @@ if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then require_expected_sha256 "external compute driver" "${DRIVER_BIN}" \ "${OPENSHELL_E2E_EXPECTED_EXTERNAL_DRIVER_SHA256:-}" env -i \ + XDG_DATA_HOME="${DRIVER_DATA_HOME}" \ OPENSHELL_COMPUTE_DRIVER_SOCKET="${DRIVER_SOCKET}" \ OPENSHELL_PODMAN_SOCKET="${OPENSHELL_PODMAN_SOCKET:-}" \ - OPENSHELL_SANDBOX_IMAGE="${SANDBOX_RUNTIME_IMAGE}" \ + OPENSHELL_SANDBOX_IMAGE="${SANDBOX_IMAGE_REQUEST}" \ OPENSHELL_SANDBOX_IMAGE_PULL_POLICY="${EXTERNAL_DRIVER_PULL_POLICY}" \ OPENSHELL_HEALTH_CHECK_INTERVAL_SECS="${EXTERNAL_DRIVER_HEALTH_CHECK_INTERVAL_SECS}" \ OPENSHELL_GRPC_ENDPOINT="${EXTERNAL_DRIVER_CALLBACK_ENDPOINT}" \ OPENSHELL_GATEWAY_PORT="${HOST_PORT}" \ OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ OPENSHELL_STOP_TIMEOUT="${PODMAN_STOP_TIMEOUT_SECS}" \ - OPENSHELL_SANDBOX_RUNTIME_IMAGE="${SANDBOX_RUNTIME_IMAGE}" \ + OPENSHELL_SANDBOX_RUNTIME_IMAGE="${SANDBOX_BOUNDARY_IMAGE}" \ OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_RUNTIME_IMAGE}" \ OPENSHELL_PODMAN_TLS_CA="${EXTERNAL_DRIVER_TLS_CA}" \ OPENSHELL_PODMAN_TLS_CERT="${EXTERNAL_DRIVER_TLS_CERT}" \ diff --git a/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py b/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py index 6e8009b759..43b3b4aa2c 100644 --- a/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py +++ b/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py @@ -22,6 +22,7 @@ IMAGE_ID = "3" * 64 IMAGE_DIGEST = f"sha256:{'4' * 64}" RUNTIME_IMAGE = f"localhost/openshell/supervisor@{IMAGE_DIGEST}" +BOUNDARY_IMAGE = f"localhost/openshell/sandbox@{IMAGE_DIGEST}" BASE_RUNTIME_IMAGE = f"docker.io/library/alpine@{IMAGE_DIGEST}" @@ -126,6 +127,7 @@ def create_variant( "sandbox_image_id": IMAGE_ID, "sandbox_image_digest": IMAGE_DIGEST, "sandbox_runtime_image": "example.invalid/sandbox@" + IMAGE_DIGEST, + "sandbox_boundary_image": BOUNDARY_IMAGE, "sandbox_client_image_alias": "example.invalid/sandbox:latest", "sandbox_client_image_alias_id": IMAGE_ID, "gateway_sha256_before_execution": result["gateway_sha256"], @@ -146,6 +148,7 @@ def create_variant( "external_driver_proxy": False, "external_driver_app_armor": False, "external_driver_environment": { + "XDG_DATA_HOME": f"/tmp/{variant}-driver-data", "OPENSHELL_COMPUTE_DRIVER_SOCKET": f"/tmp/{variant}.sock", "OPENSHELL_PODMAN_SOCKET": f"/tmp/{variant}-podman.sock", "OPENSHELL_SANDBOX_IMAGE": "example.invalid/sandbox@" + IMAGE_DIGEST, @@ -155,6 +158,7 @@ def create_variant( "OPENSHELL_GATEWAY_PORT": 18181, "OPENSHELL_NETWORK_NAME": f"{variant}-network", "OPENSHELL_STOP_TIMEOUT": 15, + "OPENSHELL_SANDBOX_RUNTIME_IMAGE": BOUNDARY_IMAGE, "OPENSHELL_SUPERVISOR_IMAGE": RUNTIME_IMAGE, "OPENSHELL_PODMAN_TLS_CA": { "path": f"/tmp/{variant}-pki/ca.crt", @@ -178,7 +182,7 @@ def create_variant( (results_dir / f"{variant}.log").write_text( f"CLI conformance run ID: fixture\n" f"gateway preflight connected: gateway=fixture, authentication=authenticated\n" - f"{lifecycle}\n{RUNTIME_IMAGE} {BASE_RUNTIME_IMAGE} example.invalid/sandbox@{IMAGE_DIGEST} example.invalid/sandbox:latest " + f"{lifecycle}\n{RUNTIME_IMAGE} {BOUNDARY_IMAGE} {BASE_RUNTIME_IMAGE} example.invalid/sandbox@{IMAGE_DIGEST} example.invalid/sandbox:latest " f'{IMAGE_ID} {IMAGE_DIGEST} {package_hash}\n"passed": true\n', encoding="utf-8", ) From 4ee9751a836ac8685de3044f809706d9b512f79b Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 13 Sep 2026 23:02:48 -0700 Subject: [PATCH 15/19] fix(podman): start sandbox before joining user namespace Signed-off-by: Drew Newberry --- crates/openshell-driver-podman/src/driver.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 542eb15b6b..ccee4bc356 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -1075,6 +1075,13 @@ impl PodmanComputeDriver { self.client .copy_to_container(&workload_id, "/sandbox", archives.workspace) .await?; + // Rootless Podman can only join another container's user + // namespace after that container has started and owns a + // live namespace. openshell-sandbox keeps the agent + // stopped until the authenticated supervisor confirms the + // boundary, so starting it here does not release workload + // execution before enforcement is established. + self.client.start_container(&workload_id).await?; specs.supervisor.join_user_namespace(&workload_id); let supervisor_id = self .client @@ -1084,10 +1091,8 @@ impl PodmanComputeDriver { self.client .copy_to_container(&supervisor_id, "/", archives.supervisor) .await?; - // Both resources and private files exist before either - // container can run. Only the trusted sandbox starts here; - // authenticated confirmation gates subsequent agent exec. - self.client.start_container(&workload_id).await?; + // The trusted sandbox is already waiting for this + // authenticated supervisor; confirmation gates agent exec. self.client.start_container(&supervisor_id).await?; Ok::<(), ComputeDriverError>(()) } @@ -3276,7 +3281,10 @@ mod tests { name: name.to_string(), namespace: String::new(), workspace: String::new(), - spec: None, + spec: Some(DriverSandboxSpec { + launch_authentication: encoded_launch_authentication(), + ..Default::default() + }), status: None, } } @@ -3490,9 +3498,9 @@ mod tests { fence_response(), StubResponse::new(StatusCode::OK, "").with_archive_members(channel_archive_members()), StubResponse::new(StatusCode::OK, "").with_archive_members(&["."]), + StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start created_response("supervisor"), StubResponse::new(StatusCode::OK, ""), // supervisor archive - StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor start ] } From bf3d8cbcde8c05dd36fba23055b42503a0b3c468 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 13 Sep 2026 23:17:43 -0700 Subject: [PATCH 16/19] fix(podman): separate supervisor user namespace Signed-off-by: Drew Newberry --- .../openshell-driver-podman/src/container.rs | 38 +++++++++++++------ crates/openshell-driver-podman/src/driver.rs | 18 +++------ 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 9d3ded6e90..1ff3ab22b3 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1392,16 +1392,6 @@ pub struct IsolationSpecs { pub supervisor: ContainerSpec, } -impl ContainerSpec { - pub(crate) fn join_user_namespace(&mut self, container_id: &str) { - self.userns = Some(UserNS { - nsmode: "container".to_string(), - value: Some(container_id.to_string()), - }); - self.idmappings = None; - } -} - pub fn build_isolation_specs( input: IsolationSpecInput<'_>, ) -> Result { @@ -1515,7 +1505,15 @@ pub fn build_isolation_specs( supervisor.cap_add.clear(); supervisor.seccomp_profile_path.clear(); // The trusted supervisor originates approved egress from the Podman host - // network. The workload remains fenced by network=none. + // network. Keep it in the caller's user namespace as well: joining the + // workload's user namespace is incompatible with host networking under + // rootless Podman, and the authenticated channel does not require a shared + // user namespace. The workload remains fenced by network=none. + supervisor.userns = Some(UserNS { + nsmode: "host".into(), + value: None, + }); + supervisor.idmappings = None; supervisor.netns.nsmode = "host".into(); supervisor.networks.clear(); supervisor.portmappings.clear(); @@ -1693,6 +1691,7 @@ mod tests { config.app_armor_profile = Some(openshell_core::config::AppArmorProfile::Localhost( "openshell-sandbox".into(), )); + config.userns = Some("auto".into()); let identity = openshell_isolation_interface::contract::ResolvedWorkloadIdentity::new( 1000, 1001, @@ -1728,6 +1727,14 @@ mod tests { assert!(spec.no_new_privileges); } assert_eq!(specs.workload.netns.nsmode, "none"); + assert_eq!( + specs + .workload + .userns + .as_ref() + .map(|userns| userns.nsmode.as_str()), + Some("auto") + ); assert_eq!( specs.workload.apparmor_profile.as_deref(), Some("openshell-sandbox") @@ -1739,6 +1746,15 @@ mod tests { assert!(specs.workload.networks.is_empty()); assert!(specs.workload.portmappings.is_empty()); assert_eq!(specs.supervisor.netns.nsmode, "host"); + assert_eq!( + specs + .supervisor + .userns + .as_ref() + .map(|userns| userns.nsmode.as_str()), + Some("host") + ); + assert!(specs.supervisor.idmappings.is_none()); assert!(specs.supervisor.networks.is_empty()); assert!(specs.supervisor.portmappings.is_empty()); assert!(specs.workload.env.is_empty()); diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index ccee4bc356..5ab33ff7dc 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -1031,7 +1031,7 @@ impl PodmanComputeDriver { tls_secrets: tls_secret_names.as_ref(), identity: &identity, }); - let mut specs = match specs { + let specs = match specs { Ok(spec) => spec, Err(e) => { cleanup_all().await; @@ -1075,14 +1075,6 @@ impl PodmanComputeDriver { self.client .copy_to_container(&workload_id, "/sandbox", archives.workspace) .await?; - // Rootless Podman can only join another container's user - // namespace after that container has started and owns a - // live namespace. openshell-sandbox keeps the agent - // stopped until the authenticated supervisor confirms the - // boundary, so starting it here does not release workload - // execution before enforcement is established. - self.client.start_container(&workload_id).await?; - specs.supervisor.join_user_namespace(&workload_id); let supervisor_id = self .client .create_typed_container(&specs.supervisor) @@ -1091,8 +1083,10 @@ impl PodmanComputeDriver { self.client .copy_to_container(&supervisor_id, "/", archives.supervisor) .await?; - // The trusted sandbox is already waiting for this - // authenticated supervisor; confirmation gates agent exec. + // Start the sandbox only after both containers and their + // bootstrap material exist. It keeps the agent stopped + // until the authenticated supervisor confirms the boundary. + self.client.start_container(&workload_id).await?; self.client.start_container(&supervisor_id).await?; Ok::<(), ComputeDriverError>(()) } @@ -3498,9 +3492,9 @@ mod tests { fence_response(), StubResponse::new(StatusCode::OK, "").with_archive_members(channel_archive_members()), StubResponse::new(StatusCode::OK, "").with_archive_members(&["."]), - StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start created_response("supervisor"), StubResponse::new(StatusCode::OK, ""), // supervisor archive + StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor start ] } From 52a03bc926f635a86292c0e42674208e74650a27 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 13 Sep 2026 23:24:22 -0700 Subject: [PATCH 17/19] fix(podman): make sandbox starts generation-aware Signed-off-by: Drew Newberry --- crates/openshell-driver-podman/src/driver.rs | 30 +++++++++++++++++-- crates/openshell-driver-podman/src/grpc.rs | 6 +++- .../openshell-driver-podman/src/isolation.rs | 8 +++-- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 5ab33ff7dc..2df6a18307 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -1061,6 +1061,7 @@ impl PodmanComputeDriver { let archives = crate::isolation::bootstrap_archives( &sandbox.id, &workload_id, + &uuid::Uuid::new_v4().to_string(), &identity, child_env, &launch_authentication, @@ -1322,9 +1323,14 @@ impl PodmanComputeDriver { pub async fn start_sandbox( &self, sandbox_id: &str, + generation_id: &str, encoded_authentication: &[u8], ) -> Result<(), ComputeDriverError> { let span_status = openshell_otel::ErrorStatusGuard::current(); + let generation = openshell_core::sandbox_generation::SandboxGenerationId::parse( + generation_id.to_string(), + ) + .map_err(|error| ComputeDriverError::InvalidArgument(error.to_string()))?; let launch_authentication = decode_launch_authentication(encoded_authentication)?; let container = self .find_container(sandbox_id) @@ -1339,7 +1345,23 @@ impl PodmanComputeDriver { .as_ref() .is_ok_and(|inspect| inspect.state.running) { - return span_status.finish(Ok(())); + let archive = self + .client + .copy_from_container( + &crate::isolation::supervisor_name(sandbox_id), + crate::isolation::RESTART_METADATA_PATH, + ) + .await?; + let bundle = + extract_first_tar_entry(&archive).map_err(ComputeDriverError::Precondition)?; + let metadata = crate::isolation::restart_metadata_from_slice(&bundle)?; + if metadata.generation == generation.as_str() { + return span_status.finish(Ok(())); + } + return span_status.finish(Err(ComputeDriverError::Precondition(format!( + "Podman sandbox is already running generation {}", + metadata.generation + )))); } self.client.stop_container(&container.id, 0).await?; self.wait_for_container_stopped(sandbox_id, &container.id) @@ -1377,6 +1399,7 @@ impl PodmanComputeDriver { let archives = crate::isolation::bootstrap_archives( sandbox_id, &container_id, + generation.as_str(), &restart_metadata.workload_identity, restart_metadata.child_env, &launch_authentication, @@ -2027,7 +2050,7 @@ mod tests { ); let authentication = encoded_launch_authentication(); test_driver(start_socket.clone()) - .start_sandbox("sandbox-1", &authentication) + .start_sandbox("sandbox-1", "generation-1", &authentication) .await .expect("start should succeed"); start_handle.await.expect("start stub should finish"); @@ -2338,7 +2361,7 @@ mod tests { ); let authentication = encoded_launch_authentication(); test_driver(start_socket.clone()) - .start_sandbox("sandbox-1", &authentication) + .start_sandbox("sandbox-1", "generation-1", &authentication) .with_subscriber(subscriber) .await .expect("start should succeed"); @@ -3349,6 +3372,7 @@ mod tests { ) .unwrap(); let bundle = serde_json::to_vec(&crate::isolation::RestartMetadata { + generation: "generation-1".to_string(), workload_identity: identity, child_env: HashMap::new(), }) diff --git a/crates/openshell-driver-podman/src/grpc.rs b/crates/openshell-driver-podman/src/grpc.rs index 94d2b31a10..b8605d9f49 100644 --- a/crates/openshell-driver-podman/src/grpc.rs +++ b/crates/openshell-driver-podman/src/grpc.rs @@ -200,7 +200,11 @@ impl ComputeDriver for ComputeDriverService { return Err(Status::invalid_argument("sandbox_id is required")); } self.driver - .start_sandbox(&request.sandbox_id, &request.launch_authentication) + .start_sandbox( + &request.sandbox_id, + &request.generation_id, + &request.launch_authentication, + ) .await .map_err(Status::from)?; Ok(Response::new(StartSandboxResponse {})) diff --git a/crates/openshell-driver-podman/src/isolation.rs b/crates/openshell-driver-podman/src/isolation.rs index 6ab0c5bbc8..3d0c0e68cc 100644 --- a/crates/openshell-driver-podman/src/isolation.rs +++ b/crates/openshell-driver-podman/src/isolation.rs @@ -134,6 +134,7 @@ pub struct BootstrapArchives { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct RestartMetadata { + pub(crate) generation: String, pub(crate) workload_identity: ResolvedWorkloadIdentity, pub(crate) child_env: HashMap, } @@ -143,6 +144,7 @@ pub struct RestartMetadata { pub fn bootstrap_archives( sandbox_id: &str, container_id: &str, + generation: &str, identity: &ResolvedWorkloadIdentity, child_env: HashMap, launch_authentication: &openshell_core::jwt::SandboxLaunchAuthentication, @@ -162,7 +164,7 @@ pub fn bootstrap_archives( network_mode: "none".into(), unexpected_networks: Vec::new(), }; - let generation = uuid::Uuid::new_v4().to_string(); + let generation = generation.to_string(); let verification_keys = launch_authentication .verification_keys .iter() @@ -196,7 +198,7 @@ pub fn bootstrap_archives( }; let runtime_descriptor = SandboxRuntimeDescriptor { boundary_id: sandbox_id.into(), - generation, + generation: generation.clone(), session_id, transport: SandboxTransport::Unix { socket_path: PathBuf::from(SOCKET_PATH), @@ -237,6 +239,7 @@ pub fn bootstrap_archives( &serde_json::to_vec(&launch_authentication.supervisor).map_err(invalid)?, )?; let restart_metadata = RestartMetadata { + generation, workload_identity: identity.clone(), child_env, }; @@ -386,6 +389,7 @@ mod tests { let archives = bootstrap_archives( "sandbox", "container", + "generation-1", &identity, child_env.clone(), &authentication, From 1d32119d80c4557179d501d651bf9dab6909cfc9 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Mon, 14 Sep 2026 00:08:26 -0700 Subject: [PATCH 18/19] fix(podman): rotate restored sandbox sessions Signed-off-by: Drew Newberry --- crates/openshell-driver-podman/src/driver.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 2df6a18307..98500b7fb2 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -1355,13 +1355,18 @@ impl PodmanComputeDriver { let bundle = extract_first_tar_entry(&archive).map_err(ComputeDriverError::Precondition)?; let metadata = crate::isolation::restart_metadata_from_slice(&bundle)?; - if metadata.generation == generation.as_str() { + if metadata.generation == generation.as_str() && encoded_authentication.is_empty() { return span_status.finish(Ok(())); } - return span_status.finish(Err(ComputeDriverError::Precondition(format!( - "Podman sandbox is already running generation {}", - metadata.generation - )))); + if metadata.generation != generation.as_str() { + return span_status.finish(Err(ComputeDriverError::Precondition(format!( + "Podman sandbox is already running generation {}", + metadata.generation + )))); + } + // A non-empty bundle for the same generation comes from + // gateway startup recovery. Restart both containers so the + // in-memory launch session changes atomically on both sides. } self.client.stop_container(&container.id, 0).await?; self.wait_for_container_stopped(sandbox_id, &container.id) From 238196fa56d5dab23133290c655c71ba717bf4e2 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Mon, 14 Sep 2026 09:23:17 -0700 Subject: [PATCH 19/19] fix(podman): bind sandbox session lineage Signed-off-by: Drew Newberry --- crates/openshell-driver-podman/src/driver.rs | 6 ++++++ .../openshell-driver-podman/src/isolation.rs | 18 ++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 98500b7fb2..1b9df795dd 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -1919,6 +1919,12 @@ mod tests { SandboxLaunchAuthentication { supervisor: SupervisorAuthBundle { session_id: openshell_core::SandboxSessionId::new(), + runtime_generation: openshell_core::sandbox_generation::SandboxGenerationId::parse( + "generation-1", + ) + .unwrap(), + session_rotation: openshell_core::jwt::SessionRotation::new(1).unwrap(), + predecessor_session_id: None, gateway_token: SecretJwt::parse("gateway.token.value").unwrap(), gateway_expires_at: i64::MAX, sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(), diff --git a/crates/openshell-driver-podman/src/isolation.rs b/crates/openshell-driver-podman/src/isolation.rs index 3d0c0e68cc..45009fb236 100644 --- a/crates/openshell-driver-podman/src/isolation.rs +++ b/crates/openshell-driver-podman/src/isolation.rs @@ -164,7 +164,10 @@ pub fn bootstrap_archives( network_mode: "none".into(), unexpected_networks: Vec::new(), }; - let generation = generation.to_string(); + let runtime_generation = launch_authentication + .supervisor + .runtime_generation + .to_string(); let verification_keys = launch_authentication .verification_keys .iter() @@ -179,8 +182,9 @@ pub fn bootstrap_archives( .collect::, _>>()?; let config = BoundaryConfig { boundary_id: sandbox_id.into(), - generation: generation.clone(), + generation: runtime_generation.clone(), session_id, + session_rotation: launch_authentication.supervisor.session_rotation, gateway_id: launch_authentication.gateway_id.clone(), verification_keys, listener: BoundaryListener::Unix { @@ -198,7 +202,7 @@ pub fn bootstrap_archives( }; let runtime_descriptor = SandboxRuntimeDescriptor { boundary_id: sandbox_id.into(), - generation: generation.clone(), + generation: runtime_generation, session_id, transport: SandboxTransport::Unix { socket_path: PathBuf::from(SOCKET_PATH), @@ -239,7 +243,7 @@ pub fn bootstrap_archives( &serde_json::to_vec(&launch_authentication.supervisor).map_err(invalid)?, )?; let restart_metadata = RestartMetadata { - generation, + generation: generation.to_string(), workload_identity: identity.clone(), child_env, }; @@ -326,6 +330,12 @@ mod tests { SandboxLaunchAuthentication { supervisor: SupervisorAuthBundle { session_id: openshell_core::SandboxSessionId::new(), + runtime_generation: openshell_core::sandbox_generation::SandboxGenerationId::parse( + "generation-1", + ) + .unwrap(), + session_rotation: openshell_core::jwt::SessionRotation::new(1).unwrap(), + predecessor_session_id: None, gateway_token: SecretJwt::parse("gateway.token.value").unwrap(), gateway_expires_at: i64::MAX, sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(),