Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 37 additions & 3 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -79,23 +79,57 @@ jobs:
! command -v mkfs.ext4 &> /dev/null || \
! command -v iptables &> /dev/null || \
! command -v qemu-system-x86_64 &> /dev/null || \
! qemu-system-x86_64 --version >/dev/null 2>&1; then
! qemu-system-x86_64 --version >/dev/null 2>&1 || \
! command -v qemu-img &> /dev/null || \
! command -v swtpm &> /dev/null || \
! test -d /usr/share/OVMF; then
apt_update_with_retry
timeout 300s sudo apt-get install -y erofs-utils e2fsprogs iptables qemu-system-x86 qemu-utils
timeout 300s sudo apt-get install -y erofs-utils e2fsprogs iptables ovmf qemu-system-x86 qemu-utils swtpm
fi
if test -f /usr/share/OVMF/OVMF_CODE_4M.secboot.fd && test -f /usr/share/OVMF/OVMF_VARS_4M.ms.fd; then
ovmf_code=/usr/share/OVMF/OVMF_CODE_4M.secboot.fd
ovmf_vars=/usr/share/OVMF/OVMF_VARS_4M.ms.fd
elif test -f /usr/share/OVMF/OVMF_CODE.secboot.fd && test -f /usr/share/OVMF/OVMF_VARS.ms.fd; then
ovmf_code=/usr/share/OVMF/OVMF_CODE.secboot.fd
ovmf_vars=/usr/share/OVMF/OVMF_VARS.ms.fd
else
echo "Secure Boot OVMF firmware with Microsoft-enrolled variables is unavailable" >&2
exit 1
fi
echo "HYPEMAN_WINDOWS_OVMF_CODE=$ovmf_code" >> "$GITHUB_ENV"
echo "HYPEMAN_WINDOWS_OVMF_VARS=$ovmf_vars" >> "$GITHUB_ENV"
go mod download

- name: Verify Linux test toolchain
run: |
set -euo pipefail
TEST_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH"
for bin in mkfs.erofs mkfs.ext4 iptables qemu-system-x86_64; do
for bin in mkfs.erofs mkfs.ext4 iptables qemu-img qemu-system-x86_64 swtpm; do
if ! sudo env "PATH=$TEST_PATH" bash -lc "command -v '$bin' >/dev/null"; then
echo "missing required binary under sudo PATH: $bin"
exit 1
fi
sudo env "PATH=$TEST_PATH" bash -lc "command -v '$bin'"
done
test -f "$HYPEMAN_WINDOWS_OVMF_CODE"
test -f "$HYPEMAN_WINDOWS_OVMF_VARS"

- name: Test Windows hypervisor primitives
run: |
TEST_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH"
for attempt in 1 2 3; do
if sudo env \
"PATH=$TEST_PATH" \
"CI=true" \
"HYPEMAN_RUN_WINDOWS_CONFIG_INTEGRATION=1" \
"HYPEMAN_WINDOWS_OVMF_CODE=$HYPEMAN_WINDOWS_OVMF_CODE" \
"HYPEMAN_WINDOWS_OVMF_VARS=$HYPEMAN_WINDOWS_OVMF_VARS" \
go test -count=1 -run '^TestWindowsConfigIntegration$' -timeout 2m ./lib/hypervisor/qemu; then
exit 0
fi
test "$attempt" = 3 || sleep 5
done
exit 1

# Slash-command runs are maintainer-approved and need authenticated pulls
# for images that are not covered by the prewarm cache.
Expand Down
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,9 @@ hypeman logs --source vmm my-app

# View Hypeman operational logs
hypeman logs --source hypeman my-app

# View software TPM logs for a TPM-backed QEMU guest
hypeman logs --source swtpm my-app
```

For all available commands, run `hypeman --help`.
Expand Down
2 changes: 2 additions & 0 deletions cmd/api/api/instances.go
Original file line number Diff line number Diff line change
Expand Up @@ -884,6 +884,8 @@ func (s *ApiService) GetInstanceLogs(ctx context.Context, request oapi.GetInstan
source = instances.LogSourceVMM
case oapi.Hypeman:
source = instances.LogSourceHypeman
case oapi.Swtpm:
source = instances.LogSourceSWTPM
}
}

Expand Down
2 changes: 2 additions & 0 deletions lib/hypervisor/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ if hv.Capabilities().SupportsSnapshot {
}
```

Capabilities also describe boot requirements such as UEFI firmware and TPM support. Guest compatibility is checked against these properties instead of hard-coding a hypervisor type. Resource requirements and image policy remain guest-level validation concerns.

## Platform Differences

### Linux (Cloud Hypervisor, QEMU)
Expand Down
7 changes: 6 additions & 1 deletion lib/hypervisor/cloudhypervisor/process.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,9 @@ func NewStarter() *Starter {
// Verify Starter implements the interface
var _ hypervisor.VMStarter = (*Starter)(nil)

func (s *Starter) ValidateConfig(hypervisor.VMConfig) error { return nil }
func (s *Starter) ValidateConfig(config hypervisor.VMConfig) error {
return hypervisor.ValidateDirectRawConfig("cloud-hypervisor", config)
}

// SocketName returns the socket filename for Cloud Hypervisor.
func (s *Starter) SocketName() string {
Expand Down Expand Up @@ -108,6 +110,9 @@ func (s *Starter) ResolveVersion(p *paths.Paths, requested string) (string, erro
// StartVM launches Cloud Hypervisor, configures the VM, and boots it.
// Returns the process ID and a Hypervisor client for subsequent operations.
func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, socketPath string, config hypervisor.VMConfig) (int, hypervisor.Hypervisor, error) {
if err := s.ValidateConfig(config); err != nil {
return 0, nil, fmt.Errorf("validate cloud-hypervisor config: %w", err)
}
log := logger.FromContext(ctx)

// Validate version
Expand Down
52 changes: 51 additions & 1 deletion lib/hypervisor/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,12 @@ type VMConfig struct {
// PCI device passthrough (GPU, etc.)
PCIDevices []string

// Boot configuration
// Boot configuration. Empty BootMode preserves the existing direct-kernel
// behavior for Linux callers.
BootMode BootMode
Firmware *FirmwareConfig
TPM *TPMConfig
Comment thread
sjmiller609 marked this conversation as resolved.

KernelPath string
InitrdPath string
KernelArgs string
Expand Down Expand Up @@ -54,9 +59,54 @@ type CPUTopology struct {
Packages int
}

type BootMode string

const (
BootModeDirect BootMode = "direct"
BootModeUEFI BootMode = "uefi"
)

// EffectiveBootMode preserves direct Linux kernel boot for existing callers.
func (c VMConfig) EffectiveBootMode() BootMode {
if c.BootMode == "" {
return BootModeDirect
}
return c.BootMode
}

// FirmwareConfig describes UEFI firmware files. CodePath is immutable firmware;
// VarsPath is per-instance writable variable storage.
type FirmwareConfig struct {
CodePath string
VarsPath string
SecureBoot bool
}

// TPMConfig describes a per-instance software TPM 2.0 endpoint.
type TPMConfig struct {
SocketPath string
StateDir string
}
Comment thread
cursor[bot] marked this conversation as resolved.

type DiskFormat string

const (
DiskFormatRaw DiskFormat = "raw"
DiskFormatQCOW2 DiskFormat = "qcow2"
)

// EffectiveFormat preserves raw disks for existing callers.
func (d DiskConfig) EffectiveFormat() DiskFormat {
if d.Format == "" {
return DiskFormatRaw
}
return d.Format
}

// DiskConfig represents a disk attached to the VM
type DiskConfig struct {
Path string
Format DiskFormat
Readonly bool
IOBps int64 // Sustained I/O rate limit in bytes/sec (0 = unlimited)
IOBurstBps int64 // Burst I/O rate in bytes/sec (0 = same as IOBps)
Expand Down
57 changes: 57 additions & 0 deletions lib/hypervisor/config_validation.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
package hypervisor

import "fmt"

// ValidateBootConfig validates boot and disk fields shared by hypervisor backends.
func ValidateBootConfig(cfg VMConfig) error {
switch cfg.EffectiveBootMode() {
case BootModeDirect:
if cfg.Firmware != nil {
return fmt.Errorf("direct boot cannot specify firmware")
}
if cfg.TPM != nil {
return fmt.Errorf("direct boot cannot specify a TPM")
}
case BootModeUEFI:
if cfg.Firmware == nil {
return fmt.Errorf("UEFI boot requires firmware")
}
if cfg.Firmware.CodePath == "" || cfg.Firmware.VarsPath == "" {
return fmt.Errorf("UEFI boot requires firmware code and variable storage paths")
}
if cfg.KernelPath != "" || cfg.InitrdPath != "" || cfg.KernelArgs != "" {
return fmt.Errorf("UEFI boot cannot specify a direct kernel, initrd, or kernel arguments")
}
if cfg.TPM != nil && (cfg.TPM.SocketPath == "" || cfg.TPM.StateDir == "") {
return fmt.Errorf("TPM requires socket and state directory paths")
}
default:
return fmt.Errorf("unsupported boot mode %q", cfg.BootMode)
}

for i, disk := range cfg.Disks {
switch disk.EffectiveFormat() {
case DiskFormatRaw, DiskFormatQCOW2:
default:
return fmt.Errorf("disk %d has unsupported format %q", i, disk.Format)
}
}
return nil
}

// ValidateDirectRawConfig preserves the Linux-only contract of backends that
// do not implement firmware boot or qcow2 disks.
func ValidateDirectRawConfig(backend string, cfg VMConfig) error {
if err := ValidateBootConfig(cfg); err != nil {
return err
}
if cfg.EffectiveBootMode() != BootModeDirect {
return fmt.Errorf("%s does not support %s boot", backend, cfg.EffectiveBootMode())
}
for i, disk := range cfg.Disks {
if disk.EffectiveFormat() != DiskFormatRaw {
return fmt.Errorf("%s does not support disk %d format %q", backend, i, disk.EffectiveFormat())
}
}
return nil
}
51 changes: 51 additions & 0 deletions lib/hypervisor/config_validation_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package hypervisor

import (
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestValidateBootConfigPreservesDirectRawDefaults(t *testing.T) {
cfg := VMConfig{
KernelPath: "/kernel",
Disks: []DiskConfig{{Path: "/rootfs"}},
}
require.NoError(t, ValidateBootConfig(cfg))
assert.Equal(t, BootModeDirect, cfg.EffectiveBootMode())
assert.Equal(t, DiskFormatRaw, cfg.Disks[0].EffectiveFormat())
}

func TestValidateBootConfigUEFI(t *testing.T) {
valid := VMConfig{
BootMode: BootModeUEFI,
Firmware: &FirmwareConfig{CodePath: "/ovmf/code", VarsPath: "/instance/vars"},
TPM: &TPMConfig{SocketPath: "/instance/swtpm.sock", StateDir: "/instance/tpm"},
Disks: []DiskConfig{{Path: "/instance/disk", Format: DiskFormatQCOW2}},
}
require.NoError(t, ValidateBootConfig(valid))

tests := []struct {
name string
cfg VMConfig
}{
{name: "missing firmware", cfg: VMConfig{BootMode: BootModeUEFI}},
{name: "direct kernel", cfg: VMConfig{BootMode: BootModeUEFI, Firmware: valid.Firmware, KernelPath: "/kernel"}},
{name: "incomplete TPM", cfg: VMConfig{BootMode: BootModeUEFI, Firmware: valid.Firmware, TPM: &TPMConfig{StateDir: "/state"}}},
{name: "unknown disk", cfg: VMConfig{Disks: []DiskConfig{{Path: "/disk", Format: "vhdx"}}}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Error(t, ValidateBootConfig(tt.cfg))
})
}
}

func TestValidateDirectRawConfigRejectsFirmwareAndQCOW2(t *testing.T) {
uefi := VMConfig{BootMode: BootModeUEFI, Firmware: &FirmwareConfig{CodePath: "/code", VarsPath: "/vars"}}
assert.ErrorContains(t, ValidateDirectRawConfig("backend", uefi), "does not support uefi boot")

qcow := VMConfig{Disks: []DiskConfig{{Path: "/disk", Format: DiskFormatQCOW2}}}
assert.ErrorContains(t, ValidateDirectRawConfig("backend", qcow), "does not support disk 0 format")
}
7 changes: 6 additions & 1 deletion lib/hypervisor/firecracker/process.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,9 @@ func WithUFFDClient(client UFFDClient) StarterOption {

var _ hypervisor.VMStarter = (*Starter)(nil)

func (s *Starter) ValidateConfig(hypervisor.VMConfig) error { return nil }
func (s *Starter) ValidateConfig(config hypervisor.VMConfig) error {
return hypervisor.ValidateDirectRawConfig("firecracker", config)
}

func (s *Starter) SocketName() string {
return "fc.sock"
Expand Down Expand Up @@ -90,6 +92,9 @@ func (s *Starter) ResolveVersion(p *paths.Paths, requested string) (string, erro
}

func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, socketPath string, config hypervisor.VMConfig) (int, hypervisor.Hypervisor, error) {
if err := s.ValidateConfig(config); err != nil {
return 0, nil, fmt.Errorf("validate firecracker config: %w", err)
}
processCtx, processSpan := hypervisor.StartProcessSpan(ctx, hypervisor.TypeFirecracker)
pid, err := s.startProcess(processCtx, p, version, socketPath)
hypervisor.FinishTraceSpan(processSpan, err)
Expand Down
6 changes: 6 additions & 0 deletions lib/hypervisor/hypervisor.go
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,12 @@ type Capabilities struct {
// SupportsVsock indicates if vsock communication is available
SupportsVsock bool

// SupportsUEFIBoot indicates if firmware boot is available.
SupportsUEFIBoot bool

// SupportsTPM indicates if a software TPM can be attached at boot.
SupportsTPM bool

// SupportsGPUPassthrough indicates if PCI device passthrough is available
SupportsGPUPassthrough bool

Expand Down
8 changes: 8 additions & 0 deletions lib/hypervisor/qemu/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,14 @@ The `qemu` backend uses `q35` on amd64 and `virt` on arm64. These architecture-n

QEMU ships many other machine models, including versioned compatibility aliases and hardware-emulation boards that do not fit Hypeman's guest contract. Hypeman intentionally does not mirror that open-ended list in its API. If another model eventually provides a useful, supportable capability profile, expose it as another hypervisor backend with explicit lifecycle and device guarantees rather than as an unchecked machine-type string.

## Firmware boot and TPM ownership

The standard amd64 `qemu` profile supports UEFI firmware boot, Secure Boot variable storage, qcow2 disks, and software TPM 2.0 devices. These requirements are represented as runtime capabilities so callers can validate a guest's requirements without branching on the QEMU type name. Direct-kernel profiles reject firmware and TPM configuration.

Firmware code is immutable host input. Each instance receives its own writable NVRAM file and TPM state directory, which must be preserved together with the guest disk. Hypeman starts `swtpm` first because QEMU connects to its control socket as a client; launching both processes concurrently would race that required ordering. Once the socket exists, QEMU starts immediately.

QEMU and `swtpm` run in detached process groups and keep running if Hypeman restarts. QEMU retains its TPM control connection, and `swtpm --terminate` exits when that connection closes. Startup cleanup owns both processes as one cleanup stack so a partial boot cannot leave either process behind. TPM output is available through the instance logs API with `source=swtpm`.

## `qemu-microvm`

The `qemu-microvm` backend uses QEMU's Linux amd64-only `microvm` board. Upstream registers this board only in the x86 system emulator; `qemu-system-aarch64` does not provide an equivalent `microvm` machine. Hypeman uses direct kernel boot, `ttyS0` serial logs, and virtio-mmio transport for disks, networking, vsock, and the optional balloon.
Expand Down
29 changes: 27 additions & 2 deletions lib/hypervisor/qemu/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,11 @@ func buildArgs(cfg hypervisor.VMConfig, machine MachineType) []string {
microvm := machine == MachineTypeMicroVM

// Machine type with KVM acceleration (arch-specific when omitted).
args = append(args, "-machine", string(machine)+",accel=kvm")
machineArg := string(machine) + ",accel=kvm"
if cfg.Firmware != nil && cfg.Firmware.SecureBoot {
machineArg += ",smm=on"
}
args = append(args, "-machine", machineArg)
if microvm {
// Do not allow a host qemu.conf to add devices outside microvm's
// documented eight virtio-mmio-device limit.
Expand Down Expand Up @@ -51,6 +55,18 @@ func buildArgs(cfg hypervisor.VMConfig, machine MachineType) []string {
args = append(args, "-device", strings.Join(balloonOpts, ","))
}

// Firmware boot. The code image is shared and immutable; variable storage is
// a per-instance writable copy.
if cfg.EffectiveBootMode() == hypervisor.BootModeUEFI {
args = append(args,
"-drive", fmt.Sprintf("if=pflash,format=raw,unit=0,file=%s,readonly=on", cfg.Firmware.CodePath),
"-drive", fmt.Sprintf("if=pflash,format=raw,unit=1,file=%s", cfg.Firmware.VarsPath),
)
if cfg.Firmware.SecureBoot {
args = append(args, "-global", "driver=cfi.pflash01,property=secure,value=on")
}
}

// Kernel and initrd
if cfg.KernelPath != "" {
args = append(args, "-kernel", cfg.KernelPath)
Expand All @@ -64,7 +80,7 @@ func buildArgs(cfg hypervisor.VMConfig, machine MachineType) []string {

// Disk configuration
for i, disk := range cfg.Disks {
driveOpts := fmt.Sprintf("file=%s,format=raw,if=none,id=drive%d", disk.Path, i)
driveOpts := fmt.Sprintf("file=%s,format=%s,if=none,id=drive%d", disk.Path, disk.EffectiveFormat(), i)
if disk.Readonly {
// Disable host-side file locking for shared readonly bases so multiple
// VMs can boot concurrently from the same image without lock contention.
Expand All @@ -80,6 +96,15 @@ func buildArgs(cfg hypervisor.VMConfig, machine MachineType) []string {
args = append(args, "-device", fmt.Sprintf("%s,drive=drive%d", virtioDevice(microvm, "virtio-blk"), i))
}

// Software TPM 2.0. The swtpm process is started by Starter before QEMU.
if cfg.TPM != nil {
args = append(args,
"-chardev", fmt.Sprintf("socket,id=chrtpm,path=%s", cfg.TPM.SocketPath),
"-tpmdev", "emulator,id=tpm0,chardev=chrtpm",
"-device", "tpm-crb,tpmdev=tpm0",
)
}

// Network configuration
for i, net := range cfg.Networks {
netdevOpts := fmt.Sprintf("tap,id=net%d,ifname=%s,script=no,downscript=no", i, net.TAPDevice)
Expand Down
Loading
Loading