feat: config-file boot for firecracker, eliminates 10 api round-trips

internal/firecracker/configfile.go

@@ -0,0 +1,143 @@
+package firecracker
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+type vmConfig struct {
+	BootSource        vmBootSource        `json:"boot-source"`
+	Drives            []vmDrive           `json:"drives"`
+	MachineConfig     vmMachineConfig     `json:"machine-config"`
+	NetworkInterfaces []vmNetworkIface    `json:"network-interfaces"`
+	Vsock             *vmVsock            `json:"vsock,omitempty"`
+	Logger            *vmLogger           `json:"logger,omitempty"`
+	MMDSConfig        *vmMMDSConfig       `json:"mmds-config,omitempty"`
+	Entropy           *vmEntropy          `json:"entropy,omitempty"`
+	Serial            *vmSerial           `json:"serial,omitempty"`
+}
+
+type vmBootSource struct {
+	KernelImagePath string `json:"kernel_image_path"`
+	BootArgs        string `json:"boot_args,omitempty"`
+}
+
+type vmDrive struct {
+	DriveID      string         `json:"drive_id"`
+	PathOnHost   string         `json:"path_on_host"`
+	IsRootDevice bool           `json:"is_root_device"`
+	IsReadOnly   bool           `json:"is_read_only"`
+	CacheType    DriveCacheType `json:"cache_type,omitempty"`
+	IOEngine     DriveIOEngine  `json:"io_engine,omitempty"`
+}
+
+type vmMachineConfig struct {
+	VcpuCount  int64 `json:"vcpu_count"`
+	MemSizeMib int64 `json:"mem_size_mib"`
+}
+
+type vmNetworkIface struct {
+	IfaceID     string `json:"iface_id"`
+	HostDevName string `json:"host_dev_name"`
+	GuestMAC    string `json:"guest_mac,omitempty"`
+}
+
+type vmVsock struct {
+	GuestCID int64  `json:"guest_cid"`
+	UDSPath  string `json:"uds_path"`
+}
+
+type vmLogger struct {
+	LogPath       string `json:"log_path"`
+	Level         string `json:"level,omitempty"`
+	ShowLevel     bool   `json:"show_level,omitempty"`
+	ShowLogOrigin bool   `json:"show_log_origin,omitempty"`
+}
+
+type vmMMDSConfig struct {
+	Version           MMDSVersion `json:"version,omitempty"`
+	NetworkInterfaces []string    `json:"network_interfaces"`
+	IPv4Address       string      `json:"ipv4_address,omitempty"`
+}
+
+type vmEntropy struct{}
+
+type vmSerial struct {
+	SerialOutPath string `json:"serial_out_path,omitempty"`
+}
+
+func writeConfigFile(chrootRootDir string, spec MachineSpec, paths machinePaths, network NetworkAllocation) (string, error) {
+	cfg := vmConfig{
+		BootSource: vmBootSource{
+			KernelImagePath: spec.KernelImagePath,
+			BootArgs:        spec.KernelArgs,
+		},
+		MachineConfig: vmMachineConfig{
+			VcpuCount:  spec.VCPUs,
+			MemSizeMib: spec.MemoryMiB,
+		},
+		Logger: &vmLogger{
+			LogPath:       paths.JailedFirecrackerLogPath,
+			Level:         defaultFirecrackerLogLevel,
+			ShowLevel:     true,
+			ShowLogOrigin: true,
+		},
+		Entropy: &vmEntropy{},
+		Serial:  &vmSerial{SerialOutPath: paths.JailedSerialLogPath},
+	}
+
+	root := spec.rootDrive()
+	cfg.Drives = append(cfg.Drives, vmDrive{
+		DriveID:      root.ID,
+		PathOnHost:   root.Path,
+		IsRootDevice: true,
+		IsReadOnly:   root.ReadOnly,
+		CacheType:    root.CacheType,
+		IOEngine:     root.IOEngine,
+	})
+	for _, drive := range spec.Drives {
+		cfg.Drives = append(cfg.Drives, vmDrive{
+			DriveID:      drive.ID,
+			PathOnHost:   drive.Path,
+			IsRootDevice: false,
+			IsReadOnly:   drive.ReadOnly,
+			CacheType:    drive.CacheType,
+			IOEngine:     drive.IOEngine,
+		})
+	}
+
+	cfg.NetworkInterfaces = append(cfg.NetworkInterfaces, vmNetworkIface{
+		IfaceID:     network.InterfaceID,
+		HostDevName: network.TapName,
+		GuestMAC:    network.GuestMAC,
+	})
+
+	if spec.MMDS != nil {
+		cfg.MMDSConfig = &vmMMDSConfig{
+			Version:           spec.MMDS.Version,
+			NetworkInterfaces: spec.MMDS.NetworkInterfaces,
+			IPv4Address:       spec.MMDS.IPv4Address,
+		}
+	}
+
+	if spec.Vsock != nil {
+		cfg.Vsock = &vmVsock{
+			GuestCID: int64(spec.Vsock.CID),
+			UDSPath:  spec.Vsock.Path,
+		}
+	}
+
+	data, err := json.Marshal(cfg)
+	if err != nil {
+		return "", fmt.Errorf("marshal vm config: %w", err)
+	}
+
+	configPath := filepath.Join(chrootRootDir, "vm_config.json")
+	if err := os.WriteFile(configPath, data, 0o644); err != nil {
+		return "", fmt.Errorf("write vm config: %w", err)
+	}
+
+	return "/vm_config.json", nil
+}

internal/firecracker/launch.go

@@ -66,7 +66,7 @@ func configureMachine(ctx context.Context, client *apiClient, paths machinePaths
 	return nil
 }
 
-func launchJailedFirecracker(paths machinePaths, machineID MachineID, firecrackerBinaryPath string, jailerBinaryPath string, enablePCI bool) (*exec.Cmd, error) {
+func launchJailedFirecracker(paths machinePaths, machineID MachineID, firecrackerBinaryPath string, jailerBinaryPath string, enablePCI bool, configFilePath string) (*exec.Cmd, error) {
 	args := []string{
 		"--id", string(machineID),
 		"--uid", strconv.Itoa(os.Getuid()),
@@ -78,10 +78,16 @@ func launchJailedFirecracker(paths machinePaths, machineID MachineID, firecracke
 		"--new-pid-ns",
 		"--",
 		"--api-sock", defaultFirecrackerSocketPath,
+	}
+	if configFilePath != "" {
+		args = append(args, "--config-file", configFilePath)
+	} else {
+		args = append(args,
 			"--log-path", paths.JailedFirecrackerLogPath,
 			"--level", defaultFirecrackerLogLevel,
 			"--show-level",
 			"--show-log-origin",
+		)
 	}
 	if enablePCI {
 		args = append(args, "--enable-pci")

internal/firecracker/launch_test.go

@@ -26,7 +26,7 @@ func TestLaunchJailedFirecrackerPassesDaemonAndLoggingFlags(t *testing.T) {
 		t.Fatalf("create log dir: %v", err)
 	}
 
-	if _, err := launchJailedFirecracker(paths, "vm-1", "/usr/bin/firecracker", jailerPath, false); err != nil {
+	if _, err := launchJailedFirecracker(paths, "vm-1", "/usr/bin/firecracker", jailerPath, false, ""); err != nil {
 		t.Fatalf("launch jailed firecracker: %v", err)
 	}
 
@@ -62,7 +62,7 @@ func TestLaunchJailedFirecrackerPassesEnablePCIWhenConfigured(t *testing.T) {
 		t.Fatalf("create log dir: %v", err)
 	}
 
-	if _, err := launchJailedFirecracker(paths, "vm-1", "/usr/bin/firecracker", jailerPath, true); err != nil {
+	if _, err := launchJailedFirecracker(paths, "vm-1", "/usr/bin/firecracker", jailerPath, true, ""); err != nil {
 		t.Fatalf("launch jailed firecracker: %v", err)
 	}
 

internal/firecracker/runtime.go

@@ -113,7 +113,19 @@ func (r *Runtime) Boot(ctx context.Context, spec MachineSpec, usedNetworks []Net
 		return nil, err
 	}
 
-	command, err := launchJailedFirecracker(paths, spec.ID, r.firecrackerBinaryPath, r.jailerBinaryPath, r.enablePCI)
+	jailedSpec, err := stageMachineFiles(spec, paths)
+	if err != nil {
+		cleanup(network, paths, nil, 0)
+		return nil, err
+	}
+
+	configFilePath, err := writeConfigFile(paths.ChrootRootDir, jailedSpec, paths, network)
+	if err != nil {
+		cleanup(network, paths, nil, 0)
+		return nil, fmt.Errorf("write config file: %w", err)
+	}
+
+	command, err := launchJailedFirecracker(paths, spec.ID, r.firecrackerBinaryPath, r.jailerBinaryPath, r.enablePCI, configFilePath)
 	if err != nil {
 		cleanup(network, paths, nil, 0)
 		return nil, err
@@ -123,23 +135,7 @@ func (r *Runtime) Boot(ctx context.Context, spec MachineSpec, usedNetworks []Net
 		cleanup(network, paths, command, 0)
 		return nil, fmt.Errorf("wait for firecracker pid: %w", err)
 	}
-
 	socketPath := procSocketPath(firecrackerPID)
-	client := newAPIClient(socketPath)
-	if err := waitForSocket(ctx, client, socketPath); err != nil {
-		cleanup(network, paths, command, firecrackerPID)
-		return nil, fmt.Errorf("wait for firecracker socket: %w", err)
-	}
-
-	jailedSpec, err := stageMachineFiles(spec, paths)
-	if err != nil {
-		cleanup(network, paths, command, firecrackerPID)
-		return nil, err
-	}
-	if err := configureMachine(ctx, client, paths, jailedSpec, network); err != nil {
-		cleanup(network, paths, command, firecrackerPID)
-		return nil, err
-	}
 
 	now := time.Now().UTC()
 	state := MachineState{
@@ -280,7 +276,7 @@ func (r *Runtime) RestoreBoot(ctx context.Context, loadSpec SnapshotLoadSpec, us
 		return nil, err
 	}
 
-	command, err := launchJailedFirecracker(paths, loadSpec.ID, r.firecrackerBinaryPath, r.jailerBinaryPath, r.enablePCI)
+	command, err := launchJailedFirecracker(paths, loadSpec.ID, r.firecrackerBinaryPath, r.jailerBinaryPath, r.enablePCI, "")
 	if err != nil {
 		cleanup(network, paths, nil, 0)
 		return nil, err