host daemon touches (#4)

* feat: launch config tests

* feat: readiness probe port alignment

internal/firecracker/api.go

@@ -33,6 +33,8 @@ type driveRequest struct {
 	PathOnHost   string `json:"path_on_host"`
 }
 
+type entropyRequest struct{}
+
 type faultResponse struct {
 	FaultMessage string `json:"fault_message"`
 }
@@ -56,6 +58,10 @@ type networkInterfaceRequest struct {
 	IfaceID     string `json:"iface_id"`
 }
 
+type serialRequest struct {
+	SerialOutPath string `json:"serial_out_path"`
+}
+
 type vsockRequest struct {
 	GuestCID int64  `json:"guest_cid"`
 	UDSPath  string `json:"uds_path"`
@@ -98,6 +104,10 @@ func (c *apiClient) PutDrive(ctx context.Context, drive driveRequest) error {
 	return c.do(ctx, http.MethodPut, endpoint, drive, nil, http.StatusNoContent)
 }
 
+func (c *apiClient) PutEntropy(ctx context.Context) error {
+	return c.do(ctx, http.MethodPut, "/entropy", entropyRequest{}, nil, http.StatusNoContent)
+}
+
 func (c *apiClient) PutMachineConfig(ctx context.Context, spec MachineSpec) error {
 	body := machineConfigRequest{
 		MemSizeMib: spec.MemoryMiB,
@@ -117,6 +127,17 @@ func (c *apiClient) PutNetworkInterface(ctx context.Context, network NetworkAllo
 	return c.do(ctx, http.MethodPut, endpoint, body, nil, http.StatusNoContent)
 }
 
+func (c *apiClient) PutSerial(ctx context.Context, serialOutPath string) error {
+	return c.do(
+		ctx,
+		http.MethodPut,
+		"/serial",
+		serialRequest{SerialOutPath: serialOutPath},
+		nil,
+		http.StatusNoContent,
+	)
+}
+
 func (c *apiClient) PutVsock(ctx context.Context, spec VsockSpec) error {
 	body := vsockRequest{
 		GuestCID: int64(spec.CID),

internal/firecracker/configure_test.go

@@ -0,0 +1,104 @@
+package firecracker
+
+import (
+	"context"
+	"io"
+	"net"
+	"net/http"
+	"path/filepath"
+	"testing"
+)
+
+type capturedRequest struct {
+	Method string
+	Path   string
+	Body   string
+}
+
+func TestConfigureMachineEnablesEntropyAndSerialBeforeStart(t *testing.T) {
+	var requests []capturedRequest
+
+	socketPath, shutdown := startUnixSocketServer(t, func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		if err != nil {
+			t.Fatalf("read request body: %v", err)
+		}
+
+		requests = append(requests, capturedRequest{
+			Method: r.Method,
+			Path:   r.URL.Path,
+			Body:   string(body),
+		})
+		w.WriteHeader(http.StatusNoContent)
+	})
+	defer shutdown()
+
+	client := newAPIClient(socketPath)
+	spec := MachineSpec{
+		ID:              "vm-1",
+		VCPUs:           1,
+		MemoryMiB:       512,
+		KernelImagePath: "/kernel",
+		RootFSPath:      "/rootfs",
+	}
+	paths := machinePaths{
+		JailedSerialLogPath: "/logs/serial.log",
+	}
+	network := NetworkAllocation{
+		InterfaceID: defaultInterfaceID,
+		TapName:     "fctap0",
+		GuestMAC:    "06:00:ac:10:00:02",
+	}
+
+	if err := configureMachine(context.Background(), client, paths, spec, network); err != nil {
+		t.Fatalf("configure machine: %v", err)
+	}
+
+	gotPaths := make([]string, 0, len(requests))
+	for _, request := range requests {
+		gotPaths = append(gotPaths, request.Path)
+	}
+	wantPaths := []string{
+		"/machine-config",
+		"/boot-source",
+		"/drives/root_drive",
+		"/network-interfaces/net0",
+		"/entropy",
+		"/serial",
+		"/actions",
+	}
+	if len(gotPaths) != len(wantPaths) {
+		t.Fatalf("request count mismatch: got %d want %d (%v)", len(gotPaths), len(wantPaths), gotPaths)
+	}
+	for i := range wantPaths {
+		if gotPaths[i] != wantPaths[i] {
+			t.Fatalf("request %d mismatch: got %q want %q", i, gotPaths[i], wantPaths[i])
+		}
+	}
+	if requests[4].Body != "{}" {
+		t.Fatalf("entropy body mismatch: got %q", requests[4].Body)
+	}
+	if requests[5].Body != "{\"serial_out_path\":\"/logs/serial.log\"}" {
+		t.Fatalf("serial body mismatch: got %q", requests[5].Body)
+	}
+}
+
+func startUnixSocketServer(t *testing.T, handler http.HandlerFunc) (string, func()) {
+	t.Helper()
+
+	socketPath := filepath.Join(t.TempDir(), "firecracker.sock")
+	listener, err := net.Listen("unix", socketPath)
+	if err != nil {
+		t.Fatalf("listen on unix socket: %v", err)
+	}
+
+	server := &http.Server{Handler: handler}
+	go func() {
+		_ = server.Serve(listener)
+	}()
+
+	return socketPath, func() {
+		_ = server.Shutdown(context.Background())
+		_ = listener.Close()
+	}
+}

internal/firecracker/launch.go

@@ -15,12 +15,13 @@ import (
 const (
 	defaultCgroupVersion           = "2"
 	defaultFirecrackerInitTimeout  = 10 * time.Second
+	defaultFirecrackerLogLevel     = "Warning"
 	defaultFirecrackerPollInterval = 10 * time.Millisecond
 	defaultRootDriveID             = "root_drive"
 	defaultVSockRunDir             = "/run"
 )
 
-func configureMachine(ctx context.Context, client *apiClient, spec MachineSpec, network NetworkAllocation) error {
+func configureMachine(ctx context.Context, client *apiClient, paths machinePaths, spec MachineSpec, network NetworkAllocation) error {
 	if err := client.PutMachineConfig(ctx, spec); err != nil {
 		return fmt.Errorf("put machine config: %w", err)
 	}
@@ -38,6 +39,12 @@ func configureMachine(ctx context.Context, client *apiClient, spec MachineSpec,
 	if err := client.PutNetworkInterface(ctx, network); err != nil {
 		return fmt.Errorf("put network interface: %w", err)
 	}
+	if err := client.PutEntropy(ctx); err != nil {
+		return fmt.Errorf("put entropy device: %w", err)
+	}
+	if err := client.PutSerial(ctx, paths.JailedSerialLogPath); err != nil {
+		return fmt.Errorf("put serial device: %w", err)
+	}
 	if spec.Vsock != nil {
 		if err := client.PutVsock(ctx, *spec.Vsock); err != nil {
 			return fmt.Errorf("put vsock: %w", err)
@@ -58,14 +65,21 @@ func launchJailedFirecracker(paths machinePaths, machineID MachineID, firecracke
 		"--exec-file", firecrackerBinaryPath,
 		"--cgroup-version", defaultCgroupVersion,
 		"--chroot-base-dir", paths.JailerBaseDir,
+		"--daemonize",
+		"--new-pid-ns",
 		"--",
 		"--api-sock", defaultFirecrackerSocketPath,
+		"--log-path", paths.JailedFirecrackerLogPath,
+		"--level", defaultFirecrackerLogLevel,
+		"--show-level",
+		"--show-log-origin",
 	)
-	command.Stdout = os.Stderr
-	command.Stderr = os.Stderr
 	if err := command.Start(); err != nil {
 		return nil, fmt.Errorf("start jailer: %w", err)
 	}
+	go func() {
+		_ = command.Wait()
+	}()
 	return command, nil
 }
 
@@ -171,7 +185,50 @@ func cleanupStartedProcess(command *exec.Cmd) {
 		return
 	}
 	_ = command.Process.Kill()
-	_ = command.Wait()
+}
+
+func readPIDFile(pidFilePath string) (int, error) {
+	payload, err := os.ReadFile(pidFilePath)
+	if err != nil {
+		return 0, err
+	}
+	pid, err := strconv.Atoi(strings.TrimSpace(string(payload)))
+	if err != nil {
+		return 0, fmt.Errorf("parse pid file %q: %w", pidFilePath, err)
+	}
+	if pid < 1 {
+		return 0, fmt.Errorf("pid file %q must contain a positive pid", pidFilePath)
+	}
+	return pid, nil
+}
+
+func waitForPIDFile(ctx context.Context, pidFilePath string) (int, error) {
+	waitContext, cancel := context.WithTimeout(ctx, defaultFirecrackerInitTimeout)
+	defer cancel()
+
+	ticker := time.NewTicker(defaultFirecrackerPollInterval)
+	defer ticker.Stop()
+
+	var lastErr error
+	for {
+		select {
+		case <-waitContext.Done():
+			if lastErr != nil {
+				return 0, fmt.Errorf("%w (pid_file=%q last_err=%v)", waitContext.Err(), pidFilePath, lastErr)
+			}
+			return 0, fmt.Errorf("%w (pid_file=%q)", waitContext.Err(), pidFilePath)
+		case <-ticker.C:
+			pid, err := readPIDFile(pidFilePath)
+			if err == nil {
+				return pid, nil
+			}
+			lastErr = err
+			if os.IsNotExist(err) {
+				continue
+			}
+			return 0, err
+		}
+	}
 }
 
 func hostVSockPath(paths machinePaths, spec MachineSpec) string {

internal/firecracker/launch_test.go

@@ -0,0 +1,95 @@
+package firecracker
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestLaunchJailedFirecrackerPassesDaemonAndLoggingFlags(t *testing.T) {
+	root := t.TempDir()
+	argsPath := filepath.Join(root, "args.txt")
+	jailerPath := filepath.Join(root, "fake-jailer.sh")
+	script := "#!/bin/sh\nprintf '%s\n' \"$@\" > " + shellQuote(argsPath) + "\n"
+	if err := os.WriteFile(jailerPath, []byte(script), 0o755); err != nil {
+		t.Fatalf("write fake jailer: %v", err)
+	}
+
+	paths, err := buildMachinePaths(root, "vm-1", "/usr/bin/firecracker")
+	if err != nil {
+		t.Fatalf("build machine paths: %v", err)
+	}
+	if err := os.MkdirAll(paths.LogDir, 0o755); err != nil {
+		t.Fatalf("create log dir: %v", err)
+	}
+
+	if _, err := launchJailedFirecracker(paths, "vm-1", "/usr/bin/firecracker", jailerPath); err != nil {
+		t.Fatalf("launch jailed firecracker: %v", err)
+	}
+
+	args := waitForFileContents(t, argsPath)
+	for _, want := range []string{
+		"--daemonize",
+		"--new-pid-ns",
+		"--log-path",
+		paths.JailedFirecrackerLogPath,
+		"--show-level",
+		"--show-log-origin",
+	} {
+		if !containsLine(args, want) {
+			t.Fatalf("missing launch argument %q in %v", want, args)
+		}
+	}
+}
+
+func TestWaitForPIDFileReadsPID(t *testing.T) {
+	pidFilePath := filepath.Join(t.TempDir(), "firecracker.pid")
+	if err := os.WriteFile(pidFilePath, []byte("4321\n"), 0o644); err != nil {
+		t.Fatalf("write pid file: %v", err)
+	}
+
+	pid, err := waitForPIDFile(context.Background(), pidFilePath)
+	if err != nil {
+		t.Fatalf("wait for pid file: %v", err)
+	}
+	if pid != 4321 {
+		t.Fatalf("pid mismatch: got %d want %d", pid, 4321)
+	}
+}
+
+func waitForFileContents(t *testing.T, path string) []string {
+	t.Helper()
+
+	timeout := time.NewTimer(2 * time.Second)
+	defer timeout.Stop()
+	ticker := time.NewTicker(10 * time.Millisecond)
+	defer ticker.Stop()
+
+	for {
+		payload, err := os.ReadFile(path)
+		if err == nil {
+			return strings.Split(strings.TrimSpace(string(payload)), "\n")
+		}
+		select {
+		case <-timeout.C:
+			t.Fatalf("timed out waiting for %q", path)
+		case <-ticker.C:
+		}
+	}
+}
+
+func containsLine(lines []string, want string) bool {
+	for _, line := range lines {
+		if line == want {
+			return true
+		}
+	}
+	return false
+}
+
+func shellQuote(value string) string {
+	return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'"
+}

internal/firecracker/paths.go

@@ -2,6 +2,7 @@ package firecracker
 
 import (
 	"fmt"
+	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -9,16 +10,25 @@ import (
 
 const (
 	defaultChrootRootDirName     = "root"
+	defaultLogDirName            = "logs"
+	defaultSerialLogName         = "serial.log"
 	defaultFirecrackerSocketDir  = "run"
+	defaultFirecrackerLogName    = "firecracker.log"
 	defaultFirecrackerSocketName = "firecracker.socket"
 	defaultFirecrackerSocketPath = "/run/firecracker.socket"
 )
 
 type machinePaths struct {
-	BaseDir       string
-	ChrootRootDir string
-	JailerBaseDir string
-	SocketPath    string
+	BaseDir                  string
+	ChrootRootDir            string
+	JailerBaseDir            string
+	LogDir                   string
+	FirecrackerLogPath       string
+	JailedFirecrackerLogPath string
+	SerialLogPath            string
+	JailedSerialLogPath      string
+	PIDFilePath              string
+	SocketPath               string
 }
 
 func buildMachinePaths(rootDir string, id MachineID, firecrackerBinaryPath string) (machinePaths, error) {
@@ -38,12 +48,19 @@ func buildMachinePaths(rootDir string, id MachineID, firecrackerBinaryPath strin
 	baseDir := filepath.Join(rootDir, "machines", string(id))
 	jailerBaseDir := filepath.Join(baseDir, "jailer")
 	chrootRootDir := filepath.Join(jailerBaseDir, binName, string(id), defaultChrootRootDirName)
+	logDir := filepath.Join(chrootRootDir, defaultLogDirName)
 
 	return machinePaths{
-		BaseDir:       baseDir,
-		ChrootRootDir: chrootRootDir,
-		JailerBaseDir: jailerBaseDir,
-		SocketPath:    filepath.Join(chrootRootDir, defaultFirecrackerSocketDir, defaultFirecrackerSocketName),
+		BaseDir:                  baseDir,
+		ChrootRootDir:            chrootRootDir,
+		JailerBaseDir:            jailerBaseDir,
+		LogDir:                   logDir,
+		FirecrackerLogPath:       filepath.Join(logDir, defaultFirecrackerLogName),
+		JailedFirecrackerLogPath: path.Join("/", defaultLogDirName, defaultFirecrackerLogName),
+		SerialLogPath:            filepath.Join(logDir, defaultSerialLogName),
+		JailedSerialLogPath:      path.Join("/", defaultLogDirName, defaultSerialLogName),
+		PIDFilePath:              filepath.Join(chrootRootDir, binName+".pid"),
+		SocketPath:               filepath.Join(chrootRootDir, defaultFirecrackerSocketDir, defaultFirecrackerSocketName),
 	}, nil
 }
 

internal/firecracker/runtime.go

@@ -69,11 +69,11 @@ func (r *Runtime) Boot(ctx context.Context, spec MachineSpec, usedNetworks []Net
 		return nil, err
 	}
 
-	cleanup := func(network NetworkAllocation, paths machinePaths, command *exec.Cmd) {
+	cleanup := func(network NetworkAllocation, paths machinePaths, command *exec.Cmd, firecrackerPID int) {
 		if preserveFailureArtifacts() {
-			fmt.Fprintf(os.Stderr, "firecracker debug: preserving failure artifacts machine=%s pid=%d socket=%s base=%s\n", spec.ID, pidOf(command), paths.SocketPath, paths.BaseDir)
 			return
 		}
+		cleanupRunningProcess(firecrackerPID)
 		cleanupStartedProcess(command)
 		_ = r.networkProvisioner.Remove(context.Background(), network)
 		if paths.BaseDir != "" {
@@ -88,55 +88,51 @@ func (r *Runtime) Boot(ctx context.Context, spec MachineSpec, usedNetworks []Net
 
 	paths, err := buildMachinePaths(r.rootDir, spec.ID, r.firecrackerBinaryPath)
 	if err != nil {
-		cleanup(network, machinePaths{}, nil)
+		cleanup(network, machinePaths{}, nil, 0)
 		return nil, err
 	}
-	if err := os.MkdirAll(paths.JailerBaseDir, 0o755); err != nil {
-		cleanup(network, paths, nil)
-		return nil, fmt.Errorf("create machine jailer dir %q: %w", paths.JailerBaseDir, err)
+	if err := os.MkdirAll(paths.LogDir, 0o755); err != nil {
+		cleanup(network, paths, nil, 0)
+		return nil, fmt.Errorf("create machine log dir %q: %w", paths.LogDir, err)
 	}
 	if err := r.networkProvisioner.Ensure(ctx, network); err != nil {
-		cleanup(network, paths, nil)
+		cleanup(network, paths, nil, 0)
 		return nil, err
 	}
 
 	command, err := launchJailedFirecracker(paths, spec.ID, r.firecrackerBinaryPath, r.jailerBinaryPath)
 	if err != nil {
-		cleanup(network, paths, nil)
+		cleanup(network, paths, nil, 0)
 		return nil, err
 	}
-	socketPath := paths.SocketPath
-	if pid := pidOf(command); pid > 0 {
-		socketPath = procSocketPath(pid)
+	firecrackerPID, err := waitForPIDFile(ctx, paths.PIDFilePath)
+	if err != nil {
+		cleanup(network, paths, command, 0)
+		return nil, fmt.Errorf("wait for firecracker pid: %w", err)
 	}
-	fmt.Fprintf(os.Stderr, "firecracker debug: launched machine=%s pid=%d socket=%s jailer_base=%s\n", spec.ID, pidOf(command), socketPath, paths.JailerBaseDir)
 
+	socketPath := procSocketPath(firecrackerPID)
 	client := newAPIClient(socketPath)
 	if err := waitForSocket(ctx, client, socketPath); err != nil {
-		cleanup(network, paths, command)
+		cleanup(network, paths, command, firecrackerPID)
 		return nil, fmt.Errorf("wait for firecracker socket: %w", err)
 	}
 
 	jailedSpec, err := stageMachineFiles(spec, paths)
 	if err != nil {
-		cleanup(network, paths, command)
+		cleanup(network, paths, command, firecrackerPID)
 		return nil, err
 	}
-	if err := configureMachine(ctx, client, jailedSpec, network); err != nil {
-		cleanup(network, paths, command)
+	if err := configureMachine(ctx, client, paths, jailedSpec, network); err != nil {
+		cleanup(network, paths, command, firecrackerPID)
 		return nil, err
 	}
 
-	pid := 0
-	if command.Process != nil {
-		pid = command.Process.Pid
-	}
-
 	now := time.Now().UTC()
 	state := MachineState{
 		ID:          spec.ID,
 		Phase:       PhaseRunning,
-		PID:         pid,
+		PID:         firecrackerPID,
 		RuntimeHost: network.GuestIP().String(),
 		SocketPath:  socketPath,
 		TapName:     network.TapName,
@@ -214,11 +210,15 @@ func processExists(pid int) bool {
 	return err == nil || err == syscall.EPERM
 }
 
-func pidOf(command *exec.Cmd) int {
-	if command == nil || command.Process == nil {
-		return 0
+func cleanupRunningProcess(pid int) {
+	if pid < 1 {
+		return
 	}
-	return command.Process.Pid
+	process, err := os.FindProcess(pid)
+	if err != nil {
+		return
+	}
+	_ = process.Kill()
 }
 
 func preserveFailureArtifacts() bool {