feat: replace neko with native GStreamer WebRTC for desktop streaming

Replace the neko binary dependency with a native GStreamer pipeline
(ximagesrc -> vp8enc -> webrtcbin) for desktop video streaming. This
removes the external neko process and integrates screen capture directly
via gstreamer-rs crate bindings behind a `desktop-gstreamer` feature flag.

Key changes:
- Add desktop_gstreamer.rs with GStreamer WebRTC pipeline management
- Rewrite signaling protocol (ready/offer/answer/candidate over WS)
- Add leaky queues and videorate for low-latency streaming
- Rewrite ICE candidates to 127.0.0.1 for Docker connectivity
- Constrain UDP port range (30000-30100) via libnice agent
- Update TypeScript SDK desktop-stream.ts for new signaling
- Update inspector DesktopTab with WebRTC Live View
- Update Dockerfiles to install GStreamer dev packages

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

CLAUDE.md

@@ -33,6 +33,12 @@
 - `docs/agent-capabilities.mdx` lists models/modes/thought levels per agent. Update it when adding a new agent or changing `fallback_config_options`. If its "Last updated" date is >2 weeks old, re-run `cd scripts/agent-configs && npx tsx dump.ts` and update the doc to match. Source data: `scripts/agent-configs/resources/*.json` and hardcoded entries in `server/packages/sandbox-agent/src/router/support.rs` (`fallback_config_options`).
 - Some agent models are gated by subscription (e.g. Claude `opus`). The live report only shows models available to the current credentials. The static doc and JSON resource files should list all known models regardless of subscription tier.
 
+## .context Directory
+
+- The `.context/` directory is gitignored and used for temporary workspace files (plans, screenshots, build stamps, etc.).
+- Never commit files from `.context/` to git.
+- Clean up temporary files (screenshots, attachments, build artifacts) from `.context/` when they are no longer needed.
+
 ## Docker Test Image
 
 - Docker-backed Rust and TypeScript tests build `docker/test-agent/Dockerfile` directly in-process and cache the image tag only in memory (`OnceLock` in Rust, module-level variable in TypeScript).

docker/runtime/Dockerfile

@@ -153,7 +153,15 @@ RUN apt-get update && apt-get install -y \
     ca-certificates \
     curl \
     git \
-    ffmpeg && \
+    ffmpeg \
+    gstreamer1.0-tools \
+    gstreamer1.0-plugins-base \
+    gstreamer1.0-plugins-good \
+    gstreamer1.0-plugins-bad \
+    gstreamer1.0-plugins-ugly \
+    gstreamer1.0-nice \
+    gstreamer1.0-x \
+    libgstreamer1.0-0 && \
     rm -rf /var/lib/apt/lists/*
 
 # Copy the binary from builder

docker/test-agent/Dockerfile

@@ -1,6 +1,16 @@
 FROM rust:1.88.0-bookworm AS builder
 WORKDIR /build
 
+# Install GStreamer dev packages for the desktop-gstreamer feature.
+RUN apt-get update -qq && \
+    apt-get install -y -qq --no-install-recommends \
+      libgstreamer1.0-dev \
+      libgstreamer-plugins-base1.0-dev \
+      libgstreamer-plugins-bad1.0-dev \
+      libnice-dev \
+      > /dev/null 2>&1 && \
+    rm -rf /var/lib/apt/lists/*
+
 COPY Cargo.toml Cargo.lock ./
 COPY server/ ./server/
 COPY gigacode/ ./gigacode/
@@ -12,7 +22,7 @@ ENV SANDBOX_AGENT_SKIP_INSPECTOR=1
 RUN --mount=type=cache,target=/usr/local/cargo/registry \
     --mount=type=cache,target=/usr/local/cargo/git \
     --mount=type=cache,target=/build/target \
-    cargo build -p sandbox-agent --release && \
+    cargo build -p sandbox-agent --release --features desktop-gstreamer && \
     cp target/release/sandbox-agent /sandbox-agent
 
 FROM node:22-bookworm-slim
@@ -26,6 +36,15 @@ RUN apt-get update -qq && \
       xdotool \
       imagemagick \
       ffmpeg \
+      gstreamer1.0-tools \
+      gstreamer1.0-plugins-base \
+      gstreamer1.0-plugins-good \
+      gstreamer1.0-plugins-bad \
+      gstreamer1.0-plugins-ugly \
+      gstreamer1.0-nice \
+      gstreamer1.0-x \
+      gstreamer1.0-pulseaudio \
+      libxcvt0 \
       x11-xserver-utils \
       dbus-x11 \
       xauth \

frontend/packages/inspector/src/components/debug/DebugPanel.tsx

@@ -80,10 +80,6 @@ const DebugPanel = ({
           <Monitor className="button-icon" style={{ marginRight: 4, width: 12, height: 12 }} />
           Desktop
         </button>
-        <button className={`debug-tab ${debugTab === "mcp" ? "active" : ""}`} onClick={() => onDebugTabChange("mcp")}>
-          <Server className="button-icon" style={{ marginRight: 4, width: 12, height: 12 }} />
-          MCP
-        </button>
         <button className={`debug-tab ${debugTab === "processes" ? "active" : ""}`} onClick={() => onDebugTabChange("processes")}>
           <Terminal className="button-icon" style={{ marginRight: 4, width: 12, height: 12 }} />
           Processes
@@ -92,6 +88,10 @@ const DebugPanel = ({
           <Play className="button-icon" style={{ marginRight: 4, width: 12, height: 12 }} />
           Run Once
         </button>
+        <button className={`debug-tab ${debugTab === "mcp" ? "active" : ""}`} onClick={() => onDebugTabChange("mcp")}>
+          <Server className="button-icon" style={{ marginRight: 4, width: 12, height: 12 }} />
+          MCP
+        </button>
         <button className={`debug-tab ${debugTab === "skills" ? "active" : ""}`} onClick={() => onDebugTabChange("skills")}>
           <Wrench className="button-icon" style={{ marginRight: 4, width: 12, height: 12 }} />
           Skills
@@ -117,13 +117,9 @@ const DebugPanel = ({
           />
         )}
 
-        {debugTab === "desktop" && (
+        {debugTab === "desktop" && <DesktopTab getClient={getClient} />}
-          <DesktopTab getClient={getClient} />
-        )}
 
-        {debugTab === "mcp" && (
+        {debugTab === "mcp" && <McpTab getClient={getClient} />}
-          <McpTab getClient={getClient} />
-        )}
 
         {debugTab === "processes" && <ProcessesTab getClient={getClient} />}
 

frontend/packages/inspector/src/components/debug/DesktopTab.tsx

@@ -1,36 +1,43 @@
-import { Loader2, Monitor, Play, RefreshCw, Square, Camera } from "lucide-react";
+import { Camera, Circle, Download, Loader2, Monitor, Play, RefreshCw, Square, Trash2, Video } from "lucide-react";
 import { useCallback, useEffect, useMemo, useState } from "react";
 import { SandboxAgentError } from "sandbox-agent";
-import type {
+import type { DesktopRecordingInfo, DesktopStatusResponse, SandboxAgent } from "sandbox-agent";
-  DesktopStatusResponse,
+import { DesktopViewer } from "@sandbox-agent/react";
-  SandboxAgent,
+import type { DesktopViewerClient } from "@sandbox-agent/react";
-} from "sandbox-agent";
-
 const MIN_SPIN_MS = 350;
-
 const extractErrorMessage = (error: unknown, fallback: string): string => {
   if (error instanceof SandboxAgentError && error.problem?.detail) return error.problem.detail;
   if (error instanceof Error) return error.message;
   return fallback;
 };
-
 const formatStartedAt = (value: string | null | undefined): string => {
-  if (!value) {
+  if (!value) return "Not started";
-    return "Not started";
-  }
   const parsed = new Date(value);
   return Number.isNaN(parsed.getTime()) ? value : parsed.toLocaleString();
 };
-
+const formatBytes = (bytes: number): string => {
+  if (bytes === 0) return "0 B";
+  const units = ["B", "KB", "MB", "GB"];
+  const i = Math.floor(Math.log(bytes) / Math.log(1024));
+  return `${(bytes / 1024 ** i).toFixed(i > 0 ? 1 : 0)} ${units[i]}`;
+};
+const formatDuration = (start: string, end?: string | null): string => {
+  const startMs = new Date(start).getTime();
+  const endMs = end ? new Date(end).getTime() : Date.now();
+  if (Number.isNaN(startMs) || Number.isNaN(endMs)) return "Unknown";
+  const seconds = Math.round((endMs - startMs) / 1000);
+  if (seconds < 60) return `${seconds}s`;
+  const mins = Math.floor(seconds / 60);
+  const secs = seconds % 60;
+  return `${mins}m ${secs}s`;
+};
 const createScreenshotUrl = async (bytes: Uint8Array): Promise<string> => {
   const payload = new Uint8Array(bytes.byteLength);
   payload.set(bytes);
   const blob = new Blob([payload.buffer], { type: "image/png" });
-
   if (typeof URL.createObjectURL === "function") {
     return URL.createObjectURL(blob);
   }
-
   return await new Promise((resolve, reject) => {
     const reader = new FileReader();
     reader.onerror = () => reject(reader.error ?? new Error("Unable to read screenshot blob."));
@@ -44,26 +51,42 @@ const createScreenshotUrl = async (bytes: Uint8Array): Promise<string> => {
     reader.readAsDataURL(blob);
   });
 };
-
+const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => {
-const DesktopTab = ({
-  getClient,
-}: {
-  getClient: () => SandboxAgent;
-}) => {
   const [status, setStatus] = useState<DesktopStatusResponse | null>(null);
   const [loading, setLoading] = useState(false);
   const [refreshing, setRefreshing] = useState(false);
   const [acting, setActing] = useState<"start" | "stop" | null>(null);
   const [error, setError] = useState<string | null>(null);
-
   const [width, setWidth] = useState("1440");
   const [height, setHeight] = useState("900");
   const [dpi, setDpi] = useState("96");
-
+  // Screenshot fallback
   const [screenshotUrl, setScreenshotUrl] = useState<string | null>(null);
   const [screenshotLoading, setScreenshotLoading] = useState(false);
   const [screenshotError, setScreenshotError] = useState<string | null>(null);
-
+  // Live view
+  const [liveViewActive, setLiveViewActive] = useState(false);
+  const [liveViewError, setLiveViewError] = useState<string | null>(null);
+  // Memoize the client as a DesktopViewerClient so the reference is stable
+  // across renders and doesn't cause the DesktopViewer effect to re-fire.
+  const viewerClient = useMemo<DesktopViewerClient>(() => {
+    const c = getClient();
+    return {
+      startDesktopStream: () => c.startDesktopStream(),
+      stopDesktopStream: () => c.stopDesktopStream(),
+      connectDesktopStream: (opts?: Parameters<SandboxAgent["connectDesktopStream"]>[0]) => c.connectDesktopStream(opts),
+    };
+  }, [getClient]);
+  // Recording
+  const [recordings, setRecordings] = useState<DesktopRecordingInfo[]>([]);
+  const [recordingLoading, setRecordingLoading] = useState(false);
+  const [recordingActing, setRecordingActing] = useState<"start" | "stop" | null>(null);
+  const [recordingError, setRecordingError] = useState<string | null>(null);
+  const [recordingFps, setRecordingFps] = useState("30");
+  const [deletingRecordingId, setDeletingRecordingId] = useState<string | null>(null);
+  const [downloadingRecordingId, setDownloadingRecordingId] = useState<string | null>(null);
+  // Active recording tracking
+  const activeRecording = useMemo(() => recordings.find((r) => r.status === "recording"), [recordings]);
   const revokeScreenshotUrl = useCallback(() => {
     setScreenshotUrl((current) => {
       if (current?.startsWith("blob:") && typeof URL.revokeObjectURL === "function") {
@@ -72,27 +95,25 @@ const DesktopTab = ({
       return null;
     });
   }, []);
-
+  const loadStatus = useCallback(
-  const loadStatus = useCallback(async (mode: "initial" | "refresh" = "initial") => {
+    async (mode: "initial" | "refresh" = "initial") => {
-    if (mode === "initial") {
+      if (mode === "initial") setLoading(true);
-      setLoading(true);
+      else setRefreshing(true);
-    } else {
+      setError(null);
-      setRefreshing(true);
+      try {
-    }
+        const next = await getClient().getDesktopStatus();
-    setError(null);
+        setStatus(next);
-    try {
+        return next;
-      const next = await getClient().getDesktopStatus();
+      } catch (loadError) {
-      setStatus(next);
+        setError(extractErrorMessage(loadError, "Unable to load desktop status."));
-      return next;
+        return null;
-    } catch (loadError) {
+      } finally {
-      setError(extractErrorMessage(loadError, "Unable to load desktop status."));
+        setLoading(false);
-      return null;
+        setRefreshing(false);
-    } finally {
+      }
-      setLoading(false);
+    },
-      setRefreshing(false);
+    [getClient],
-    }
+  );
-  }, [getClient]);
-
   const refreshScreenshot = useCallback(async () => {
     setScreenshotLoading(true);
     setScreenshotError(null);
@@ -107,25 +128,38 @@ const DesktopTab = ({
       setScreenshotLoading(false);
     }
   }, [getClient, revokeScreenshotUrl]);
-
+  const loadRecordings = useCallback(async () => {
+    setRecordingLoading(true);
+    setRecordingError(null);
+    try {
+      const result = await getClient().listDesktopRecordings();
+      setRecordings(result.recordings);
+    } catch (loadError) {
+      setRecordingError(extractErrorMessage(loadError, "Unable to load recordings."));
+    } finally {
+      setRecordingLoading(false);
+    }
+  }, [getClient]);
   useEffect(() => {
     void loadStatus();
   }, [loadStatus]);
-
   useEffect(() => {
     if (status?.state === "active") {
-      void refreshScreenshot();
+      void loadRecordings();
     } else {
       revokeScreenshotUrl();
+      setLiveViewActive(false);
     }
-  }, [refreshScreenshot, revokeScreenshotUrl, status?.state]);
+  }, [status?.state, loadRecordings, revokeScreenshotUrl]);
-
   useEffect(() => {
-    return () => {
+    return () => revokeScreenshotUrl();
-      revokeScreenshotUrl();
-    };
   }, [revokeScreenshotUrl]);
-
+  // Poll recording list while a recording is active
+  useEffect(() => {
+    if (!activeRecording) return;
+    const interval = setInterval(() => void loadRecordings(), 3000);
+    return () => clearInterval(interval);
+  }, [activeRecording, loadRecordings]);
   const handleStart = async () => {
     const parsedWidth = Number.parseInt(width, 10);
     const parsedHeight = Number.parseInt(height, 10);
@@ -140,9 +174,6 @@ const DesktopTab = ({
         dpi: Number.isFinite(parsedDpi) ? parsedDpi : undefined,
       });
       setStatus(next);
-      if (next.state === "active") {
-        await refreshScreenshot();
-      }
     } catch (startError) {
       setError(extractErrorMessage(startError, "Unable to start desktop runtime."));
       await loadStatus("refresh");
@@ -154,7 +185,6 @@ const DesktopTab = ({
       setActing(null);
     }
   };
-
   const handleStop = async () => {
     setActing("stop");
     setError(null);
@@ -163,6 +193,7 @@ const DesktopTab = ({
       const next = await getClient().stopDesktop();
       setStatus(next);
       revokeScreenshotUrl();
+      setLiveViewActive(false);
     } catch (stopError) {
       setError(extractErrorMessage(stopError, "Unable to stop desktop runtime."));
       await loadStatus("refresh");
@@ -174,62 +205,102 @@ const DesktopTab = ({
       setActing(null);
     }
   };
-
+  const handleStartRecording = async () => {
+    const fps = Number.parseInt(recordingFps, 10);
+    setRecordingActing("start");
+    setRecordingError(null);
+    try {
+      await getClient().startDesktopRecording({
+        fps: Number.isFinite(fps) && fps > 0 ? fps : undefined,
+      });
+      await loadRecordings();
+    } catch (err) {
+      setRecordingError(extractErrorMessage(err, "Unable to start recording."));
+    } finally {
+      setRecordingActing(null);
+    }
+  };
+  const handleStopRecording = async () => {
+    setRecordingActing("stop");
+    setRecordingError(null);
+    try {
+      await getClient().stopDesktopRecording();
+      await loadRecordings();
+    } catch (err) {
+      setRecordingError(extractErrorMessage(err, "Unable to stop recording."));
+    } finally {
+      setRecordingActing(null);
+    }
+  };
+  const handleDeleteRecording = async (id: string) => {
+    setDeletingRecordingId(id);
+    try {
+      await getClient().deleteDesktopRecording(id);
+      setRecordings((prev) => prev.filter((r) => r.id !== id));
+    } catch (err) {
+      setRecordingError(extractErrorMessage(err, "Unable to delete recording."));
+    } finally {
+      setDeletingRecordingId(null);
+    }
+  };
+  const handleDownloadRecording = async (id: string, fileName: string) => {
+    setDownloadingRecordingId(id);
+    try {
+      const bytes = await getClient().downloadDesktopRecording(id);
+      const blob = new Blob([bytes], { type: "video/mp4" });
+      const url = URL.createObjectURL(blob);
+      const a = document.createElement("a");
+      a.href = url;
+      a.download = fileName;
+      document.body.appendChild(a);
+      a.click();
+      document.body.removeChild(a);
+      URL.revokeObjectURL(url);
+    } catch (err) {
+      setRecordingError(extractErrorMessage(err, "Unable to download recording."));
+    } finally {
+      setDownloadingRecordingId(null);
+    }
+  };
   const canRefreshScreenshot = status?.state === "active";
+  const isActive = status?.state === "active";
   const resolutionLabel = useMemo(() => {
     const resolution = status?.resolution;
     if (!resolution) return "Unknown";
     const dpiLabel = resolution.dpi ? ` @ ${resolution.dpi} DPI` : "";
     return `${resolution.width} x ${resolution.height}${dpiLabel}`;
   }, [status?.resolution]);
-
   return (
     <div className="desktop-panel">
       <div className="inline-row" style={{ marginBottom: 16 }}>
-        <button
+        <button className="button secondary small" onClick={() => void loadStatus("refresh")} disabled={loading || refreshing}>
-          className="button secondary small"
-          onClick={() => void loadStatus("refresh")}
-          disabled={loading || refreshing}
-        >
           <RefreshCw className={`button-icon ${loading || refreshing ? "spinner-icon" : ""}`} />
           Refresh Status
         </button>
-        <button
+        {isActive && !liveViewActive && (
-          className="button secondary small"
+          <button className="button secondary small" onClick={() => void refreshScreenshot()} disabled={!canRefreshScreenshot || screenshotLoading}>
-          onClick={() => void refreshScreenshot()}
+            {screenshotLoading ? <Loader2 className="button-icon spinner-icon" /> : <Camera className="button-icon" />}
-          disabled={!canRefreshScreenshot || screenshotLoading}
+            Screenshot
-        >
+          </button>
-          {screenshotLoading ? (
+        )}
-            <Loader2 className="button-icon spinner-icon" />
-          ) : (
-            <Camera className="button-icon" />
-          )}
-          Refresh Screenshot
-        </button>
       </div>
-
       {error && <div className="banner error">{error}</div>}
       {screenshotError && <div className="banner error">{screenshotError}</div>}
-
+      {/* ========== Runtime Section ========== */}
       <div className="card">
         <div className="card-header">
           <span className="card-title">
             <Monitor size={14} style={{ marginRight: 6 }} />
             Desktop Runtime
           </span>
-          <span className={`pill ${
+          <span
-            status?.state === "active"
+            className={`pill ${
-              ? "success"
+              status?.state === "active" ? "success" : status?.state === "install_required" ? "warning" : status?.state === "failed" ? "danger" : ""
-              : status?.state === "install_required"
+            }`}
-                ? "warning"
+          >
-                : status?.state === "failed"
-                  ? "danger"
-                  : ""
-          }`}>
             {status?.state ?? "unknown"}
           </span>
         </div>
-
         <div className="desktop-state-grid">
           <div>
             <div className="card-meta">Display</div>
@@ -244,65 +315,35 @@ const DesktopTab = ({
             <div>{formatStartedAt(status?.startedAt)}</div>
           </div>
         </div>
-
         <div className="desktop-start-controls">
           <div className="desktop-input-group">
             <label className="label">Width</label>
-            <input
+            <input className="setup-input mono" value={width} onChange={(event) => setWidth(event.target.value)} inputMode="numeric" />
-              className="setup-input mono"
-              value={width}
-              onChange={(event) => setWidth(event.target.value)}
-              inputMode="numeric"
-            />
           </div>
           <div className="desktop-input-group">
             <label className="label">Height</label>
-            <input
+            <input className="setup-input mono" value={height} onChange={(event) => setHeight(event.target.value)} inputMode="numeric" />
-              className="setup-input mono"
-              value={height}
-              onChange={(event) => setHeight(event.target.value)}
-              inputMode="numeric"
-            />
           </div>
           <div className="desktop-input-group">
             <label className="label">DPI</label>
-            <input
+            <input className="setup-input mono" value={dpi} onChange={(event) => setDpi(event.target.value)} inputMode="numeric" />
-              className="setup-input mono"
-              value={dpi}
-              onChange={(event) => setDpi(event.target.value)}
-              inputMode="numeric"
-            />
           </div>
         </div>
-
         <div className="card-actions">
-          <button
+          {isActive ? (
-            className="button success small"
+            <button className="button danger small" onClick={() => void handleStop()} disabled={acting === "stop"}>
-            onClick={() => void handleStart()}
+              {acting === "stop" ? <Loader2 className="button-icon spinner-icon" /> : <Square className="button-icon" />}
-            disabled={acting === "start"}
+              Stop Desktop
-          >
+            </button>
-            {acting === "start" ? (
+          ) : (
-              <Loader2 className="button-icon spinner-icon" />
+            <button className="button success small" onClick={() => void handleStart()} disabled={acting === "start"}>
-            ) : (
+              {acting === "start" ? <Loader2 className="button-icon spinner-icon" /> : <Play className="button-icon" />}
-              <Play className="button-icon" />
+              Start Desktop
-            )}
+            </button>
-            Start Desktop
+          )}
-          </button>
-          <button
-            className="button danger small"
-            onClick={() => void handleStop()}
-            disabled={acting === "stop"}
-          >
-            {acting === "stop" ? (
-              <Loader2 className="button-icon spinner-icon" />
-            ) : (
-              <Square className="button-icon" />
-            )}
-            Stop Desktop
-          </button>
         </div>
       </div>
-
+      {/* ========== Missing Dependencies ========== */}
       {status?.missingDependencies && status.missingDependencies.length > 0 && (
         <div className="card">
           <div className="card-header">
@@ -310,18 +351,188 @@ const DesktopTab = ({
           </div>
           <div className="desktop-chip-list">
             {status.missingDependencies.map((dependency) => (
-              <span key={dependency} className="pill warning">{dependency}</span>
+              <span key={dependency} className="pill warning">
+                {dependency}
+              </span>
             ))}
           </div>
           {status.installCommand && (
             <>
-              <div className="card-meta" style={{ marginTop: 12 }}>Install command</div>
+              <div className="card-meta" style={{ marginTop: 12 }}>
+                Install command
+              </div>
               <div className="mono desktop-command">{status.installCommand}</div>
             </>
           )}
         </div>
       )}
-
+      {/* ========== Live View Section ========== */}
+      <div className="card">
+        <div className="card-header">
+          <span className="card-title">
+            <Video size={14} style={{ marginRight: 6 }} />
+            Live View
+          </span>
+          {isActive && (
+            <button
+              className={`button small ${liveViewActive ? "danger" : "success"}`}
+              onClick={(e) => {
+                e.stopPropagation();
+                if (liveViewActive) {
+                  // Stop: close viewer then stop the stream process
+                  setLiveViewActive(false);
+                  void getClient()
+                    .stopDesktopStream()
+                    .catch(() => undefined);
+                } else {
+                  setLiveViewActive(true);
+                }
+              }}
+              style={{ padding: "4px 10px", fontSize: 11 }}
+            >
+              {liveViewActive ? (
+                <>
+                  <Square size={12} style={{ marginRight: 4 }} />
+                  Stop Stream
+                </>
+              ) : (
+                <>
+                  <Play size={12} style={{ marginRight: 4 }} />
+                  Start Stream
+                </>
+              )}
+            </button>
+          )}
+        </div>
+        {liveViewError && (
+          <div className="banner error" style={{ marginBottom: 8 }}>
+            {liveViewError}
+          </div>
+        )}
+        {!isActive && <div className="desktop-screenshot-empty">Start the desktop runtime to enable live view.</div>}
+        {isActive && liveViewActive && <DesktopViewer client={viewerClient} autoStart={true} showStatusBar={true} />}
+        {isActive && !liveViewActive && (
+          <>
+            {screenshotUrl ? (
+              <div className="desktop-screenshot-frame">
+                <img src={screenshotUrl} alt="Desktop screenshot" className="desktop-screenshot-image" />
+              </div>
+            ) : (
+              <div className="desktop-screenshot-empty">Click "Start Stream" for live desktop view, or use the Screenshot button above.</div>
+            )}
+          </>
+        )}
+      </div>
+      {/* ========== Recording Section ========== */}
+      <div className="card">
+        <div className="card-header">
+          <span className="card-title">
+            <Circle size={14} style={{ marginRight: 6, fill: activeRecording ? "#ff3b30" : "none" }} />
+            Recording
+          </span>
+          {activeRecording && <span className="pill danger">Recording</span>}
+        </div>
+        {recordingError && (
+          <div className="banner error" style={{ marginBottom: 8 }}>
+            {recordingError}
+          </div>
+        )}
+        {!isActive && <div className="desktop-screenshot-empty">Start the desktop runtime to enable recording.</div>}
+        {isActive && (
+          <>
+            <div className="desktop-start-controls" style={{ gridTemplateColumns: "1fr" }}>
+              <div className="desktop-input-group">
+                <label className="label">FPS</label>
+                <input
+                  className="setup-input mono"
+                  value={recordingFps}
+                  onChange={(e) => setRecordingFps(e.target.value)}
+                  inputMode="numeric"
+                  style={{ maxWidth: 80 }}
+                  disabled={!!activeRecording}
+                />
+              </div>
+            </div>
+            <div className="card-actions">
+              {!activeRecording ? (
+                <button className="button danger small" onClick={() => void handleStartRecording()} disabled={recordingActing === "start"}>
+                  {recordingActing === "start" ? (
+                    <Loader2 className="button-icon spinner-icon" />
+                  ) : (
+                    <Circle size={14} className="button-icon" style={{ fill: "#ff3b30" }} />
+                  )}
+                  Start Recording
+                </button>
+              ) : (
+                <button className="button secondary small" onClick={() => void handleStopRecording()} disabled={recordingActing === "stop"}>
+                  {recordingActing === "stop" ? <Loader2 className="button-icon spinner-icon" /> : <Square className="button-icon" />}
+                  Stop Recording
+                </button>
+              )}
+              <button className="button secondary small" onClick={() => void loadRecordings()} disabled={recordingLoading}>
+                <RefreshCw className={`button-icon ${recordingLoading ? "spinner-icon" : ""}`} />
+                Refresh
+              </button>
+            </div>
+            {recordings.length > 0 && (
+              <div className="desktop-process-list" style={{ marginTop: 12 }}>
+                {recordings.map((rec) => (
+                  <div key={rec.id} className="desktop-process-item">
+                    <div style={{ display: "flex", alignItems: "center", justifyContent: "space-between" }}>
+                      <div>
+                        <strong className="mono" style={{ fontSize: 12 }}>
+                          {rec.fileName}
+                        </strong>
+                        <span
+                          className={`pill ${rec.status === "recording" ? "danger" : rec.status === "completed" ? "success" : "warning"}`}
+                          style={{ marginLeft: 8 }}
+                        >
+                          {rec.status}
+                        </span>
+                      </div>
+                      {rec.status === "completed" && (
+                        <div style={{ display: "flex", gap: 4 }}>
+                          <button
+                            className="button ghost small"
+                            title="Download"
+                            onClick={() => void handleDownloadRecording(rec.id, rec.fileName)}
+                            disabled={downloadingRecordingId === rec.id}
+                            style={{ padding: "4px 6px" }}
+                          >
+                            {downloadingRecordingId === rec.id ? <Loader2 size={14} className="spinner-icon" /> : <Download size={14} />}
+                          </button>
+                          <button
+                            className="button ghost small"
+                            title="Delete"
+                            onClick={() => void handleDeleteRecording(rec.id)}
+                            disabled={deletingRecordingId === rec.id}
+                            style={{ padding: "4px 6px", color: "var(--danger)" }}
+                          >
+                            {deletingRecordingId === rec.id ? <Loader2 size={14} className="spinner-icon" /> : <Trash2 size={14} />}
+                          </button>
+                        </div>
+                      )}
+                    </div>
+                    <div className="mono" style={{ fontSize: 11, color: "var(--muted)", marginTop: 4 }}>
+                      {formatBytes(rec.bytes)}
+                      {" \u00b7 "}
+                      {formatDuration(rec.startedAt, rec.endedAt)}
+                      {" \u00b7 "}
+                      {formatStartedAt(rec.startedAt)}
+                    </div>
+                  </div>
+                ))}
+              </div>
+            )}
+            {recordings.length === 0 && !recordingLoading && (
+              <div className="desktop-screenshot-empty" style={{ marginTop: 8 }}>
+                No recordings yet. Click "Start Recording" to begin.
+              </div>
+            )}
+          </>
+        )}
+      </div>
+      {/* ========== Diagnostics Section ========== */}
       {(status?.lastError || status?.runtimeLogPath || (status?.processes?.length ?? 0) > 0) && (
         <div className="card">
           <div className="card-header">
@@ -352,9 +563,7 @@ const DesktopTab = ({
                         {process.running ? "running" : "stopped"}
                       </span>
                     </div>
-                    <div className="mono">
+                    <div className="mono">{process.pid ? `pid ${process.pid}` : "no pid"}</div>
-                      {process.pid ? `pid ${process.pid}` : "no pid"}
-                    </div>
                     {process.logPath && <div className="mono">{process.logPath}</div>}
                   </div>
                 ))}
@@ -363,31 +572,7 @@ const DesktopTab = ({
           )}
         </div>
       )}
-
-      <div className="card">
-        <div className="card-header">
-          <span className="card-title">Latest Screenshot</span>
-          {status?.state === "active" ? (
-            <span className="card-meta">Manual refresh only</span>
-          ) : null}
-        </div>
-
-        {loading ? <div className="card-meta">Loading...</div> : null}
-        {!loading && !screenshotUrl && (
-          <div className="desktop-screenshot-empty">
-            {status?.state === "active"
-              ? "No screenshot loaded yet."
-              : "Start the desktop runtime to capture a screenshot."}
-          </div>
-        )}
-        {screenshotUrl && (
-          <div className="desktop-screenshot-frame">
-            <img src={screenshotUrl} alt="Desktop screenshot" className="desktop-screenshot-image" />
-          </div>
-        )}
-      </div>
     </div>
   );
 };
-
 export default DesktopTab;

frontend/packages/inspector/vite.config.ts

@@ -8,7 +8,7 @@ export default defineConfig(({ command }) => ({
     port: 5173,
     proxy: {
       "/v1": {
-        target: "http://localhost:2468",
+        target: process.env.SANDBOX_AGENT_URL || "http://localhost:2468",
         changeOrigin: true,
         ws: true,
       },

justfile

@@ -76,6 +76,26 @@ run-gigacode *ARGS:
 dev-docs:
 	cd docs && pnpm dlx mintlify dev --host 0.0.0.0
 
+# Start the desktop dev stack (sandbox-agent backend in Docker + inspector frontend)
+[group('server')]
+server-dev:
+	docker compose -f server/compose.dev.yaml up --build --force-recreate -d
+
+# Stop the desktop dev stack
+[group('server')]
+server-dev-down:
+	docker compose -f server/compose.dev.yaml down
+
+# Tail desktop dev stack logs
+[group('server')]
+server-dev-logs *ARGS:
+	docker compose -f server/compose.dev.yaml logs -f --tail=200 {{ ARGS }}
+
+# Rebuild and restart only the backend container
+[group('server')]
+server-dev-restart-backend:
+	docker compose -f server/compose.dev.yaml up --build --force-recreate -d backend
+
 install:
     pnpm install
     pnpm build --filter @sandbox-agent/inspector...

sdks/react/src/DesktopViewer.tsx

@@ -2,26 +2,19 @@
 
 import type { CSSProperties, MouseEvent, WheelEvent } from "react";
 import { useEffect, useRef, useState } from "react";
-import type {
+import type { DesktopMouseButton, DesktopStreamErrorStatus, DesktopStreamReadyStatus, DesktopStreamSession, SandboxAgent } from "sandbox-agent";
-  DesktopMouseButton,
-  DesktopStreamErrorStatus,
-  DesktopStreamReadyStatus,
-  SandboxAgent,
-} from "sandbox-agent";
 
 type ConnectionState = "connecting" | "ready" | "closed" | "error";
 
-export type DesktopViewerClient = Pick<
+export type DesktopViewerClient = Pick<SandboxAgent, "startDesktopStream" | "stopDesktopStream" | "connectDesktopStream">;
-  SandboxAgent,
-  "startDesktopStream" | "stopDesktopStream" | "connectDesktopStream"
->;
 
 export interface DesktopViewerProps {
   client: DesktopViewerClient;
   className?: string;
   style?: CSSProperties;
-  imageStyle?: CSSProperties;
+  autoStart?: boolean;
-  height?: number | string;
+  showStatusBar?: boolean;
+  tabIndex?: number;
   onConnect?: (status: DesktopStreamReadyStatus) => void;
   onDisconnect?: () => void;
   onError?: (error: DesktopStreamErrorStatus | Error) => void;
@@ -31,11 +24,7 @@ const shellStyle: CSSProperties = {
   display: "flex",
   flexDirection: "column",
   overflow: "hidden",
-  border: "1px solid rgba(15, 23, 42, 0.14)",
+  width: "100%",
-  borderRadius: 14,
-  background:
-    "linear-gradient(180deg, rgba(248, 250, 252, 0.96) 0%, rgba(226, 232, 240, 0.92) 100%)",
-  boxShadow: "0 20px 40px rgba(15, 23, 42, 0.08)",
 };
 
 const statusBarStyle: CSSProperties = {
@@ -44,28 +33,22 @@ const statusBarStyle: CSSProperties = {
   justifyContent: "space-between",
   gap: 12,
   padding: "10px 14px",
-  borderBottom: "1px solid rgba(15, 23, 42, 0.08)",
-  background: "rgba(255, 255, 255, 0.78)",
-  color: "#0f172a",
   fontSize: 12,
   lineHeight: 1.4,
 };
 
 const viewportStyle: CSSProperties = {
   position: "relative",
-  display: "flex",
+  width: "100%",
-  alignItems: "center",
-  justifyContent: "center",
   overflow: "hidden",
-  background:
+  background: "#000",
-    "radial-gradient(circle at top, rgba(14, 165, 233, 0.18), transparent 45%), linear-gradient(180deg, #0f172a 0%, #111827 100%)",
+  outline: "none",
 };
 
-const imageBaseStyle: CSSProperties = {
+const videoBaseStyle: CSSProperties = {
   display: "block",
   width: "100%",
-  height: "100%",
+  height: "auto",
-  objectFit: "contain",
   userSelect: "none",
 };
 
@@ -90,90 +73,96 @@ export const DesktopViewer = ({
   client,
   className,
   style,
-  imageStyle,
+  autoStart = true,
-  height = 480,
+  showStatusBar = true,
+  tabIndex = 0,
   onConnect,
   onDisconnect,
   onError,
 }: DesktopViewerProps) => {
+  const videoRef = useRef<HTMLVideoElement | null>(null);
   const wrapperRef = useRef<HTMLDivElement | null>(null);
-  const sessionRef = useRef<ReturnType<DesktopViewerClient["connectDesktopStream"]> | null>(null);
+  const sessionRef = useRef<DesktopStreamSession | null>(null);
-  const [connectionState, setConnectionState] = useState<ConnectionState>("connecting");
+  const [connectionState, setConnectionState] = useState<ConnectionState>(autoStart ? "connecting" : "closed");
-  const [statusMessage, setStatusMessage] = useState("Starting desktop stream...");
+  const [statusMessage, setStatusMessage] = useState(autoStart ? "Starting desktop stream..." : "Stream not started.");
-  const [frameUrl, setFrameUrl] = useState<string | null>(null);
   const [resolution, setResolution] = useState<{ width: number; height: number } | null>(null);
 
-  useEffect(() => {
+  // Store callbacks and client in refs to keep them out of the effect deps.
-    let cancelled = false;
+  const onConnectRef = useRef(onConnect);
-    let lastObjectUrl: string | null = null;
+  onConnectRef.current = onConnect;
-    let session: ReturnType<DesktopViewerClient["connectDesktopStream"]> | null = null;
+  const onDisconnectRef = useRef(onDisconnect);
+  onDisconnectRef.current = onDisconnect;
+  const onErrorRef = useRef(onError);
+  onErrorRef.current = onError;
+  const clientRef = useRef(client);
+  clientRef.current = client;
 
+  useEffect(() => {
+    if (!autoStart) {
+      setConnectionState("closed");
+      setStatusMessage("Stream not started.");
+      return;
+    }
+
+    let cancelled = false;
     setConnectionState("connecting");
     setStatusMessage("Starting desktop stream...");
     setResolution(null);
 
+    const cl = clientRef.current;
+
     const connect = async () => {
       try {
-        await client.startDesktopStream();
+        await cl.startDesktopStream();
-        if (cancelled) {
+        if (cancelled) return;
-          return;
-        }
 
-        session = client.connectDesktopStream();
+        const session = cl.connectDesktopStream();
         sessionRef.current = session;
+
         session.onReady((status) => {
-          if (cancelled) {
+          if (cancelled) return;
-            return;
+          setResolution({ width: status.width, height: status.height });
+          setStatusMessage("Negotiating WebRTC...");
+          onConnectRef.current?.(status);
+        });
+
+        session.onTrack((stream) => {
+          if (cancelled) return;
+          if (videoRef.current) {
+            videoRef.current.srcObject = stream;
           }
           setConnectionState("ready");
           setStatusMessage("Desktop stream connected.");
-          setResolution({ width: status.width, height: status.height });
+          // Grab keyboard focus when connected.
-          onConnect?.(status);
+          wrapperRef.current?.focus();
         });
-        session.onFrame((frame) => {
+
-          if (cancelled) {
+        session.onConnect(() => {
-            return;
+          if (cancelled) return;
-          }
+          setConnectionState("ready");
-          const nextUrl = URL.createObjectURL(
+          setStatusMessage("Desktop stream connected.");
-            new Blob([frame.slice().buffer], { type: "image/jpeg" }),
+          wrapperRef.current?.focus();
-          );
-          setFrameUrl((current) => {
-            if (current) {
-              URL.revokeObjectURL(current);
-            }
-            return nextUrl;
-          });
-          if (lastObjectUrl) {
-            URL.revokeObjectURL(lastObjectUrl);
-          }
-          lastObjectUrl = nextUrl;
         });
+
         session.onError((error) => {
-          if (cancelled) {
+          if (cancelled) return;
-            return;
-          }
           setConnectionState("error");
           setStatusMessage(error instanceof Error ? error.message : error.message);
-          onError?.(error);
+          onErrorRef.current?.(error);
         });
-        session.onClose(() => {
+
-          if (cancelled) {
+        session.onDisconnect(() => {
-            return;
+          if (cancelled) return;
-          }
+          setConnectionState((cur) => (cur === "error" ? cur : "closed"));
-          setConnectionState((current) => (current === "error" ? current : "closed"));
+          setStatusMessage((cur) => (cur === "Desktop stream connected." ? "Desktop stream disconnected." : cur));
-          setStatusMessage((current) =>
+          onDisconnectRef.current?.();
-            current === "Desktop stream connected." ? "Desktop stream disconnected." : current,
-          );
-          onDisconnect?.();
         });
       } catch (error) {
-        if (cancelled) {
+        if (cancelled) return;
-          return;
+        const nextError = error instanceof Error ? error : new Error("Failed to start desktop stream.");
-        }
-        const nextError = error instanceof Error ? error : new Error("Failed to initialize desktop stream.");
         setConnectionState("error");
         setStatusMessage(nextError.message);
-        onError?.(nextError);
+        onErrorRef.current?.(nextError);
       }
     };
 
@@ -181,36 +170,28 @@ export const DesktopViewer = ({
 
     return () => {
       cancelled = true;
-      session?.close();
+      const session = sessionRef.current;
-      sessionRef.current = null;
+      if (session) {
-      void client.stopDesktopStream().catch(() => undefined);
+        session.close();
-      setFrameUrl((current) => {
+        sessionRef.current = null;
-        if (current) {
-          URL.revokeObjectURL(current);
-        }
-        return null;
-      });
-      if (lastObjectUrl) {
-        URL.revokeObjectURL(lastObjectUrl);
       }
+      if (videoRef.current) {
+        videoRef.current.srcObject = null;
+      }
+      // Note: we do NOT call stopDesktopStream() here. The parent component
+      // manages the stream lifecycle. Calling stop on unmount would kill the
+      // streaming process and race with subsequent mounts.
     };
-  }, [client, onConnect, onDisconnect, onError]);
+  }, [autoStart]);
 
   const scalePoint = (clientX: number, clientY: number) => {
-    const wrapper = wrapperRef.current;
+    const video = videoRef.current;
-    if (!wrapper || !resolution) {
+    if (!video || !resolution) return null;
-      return null;
+    const rect = video.getBoundingClientRect();
-    }
+    if (rect.width === 0 || rect.height === 0) return null;
-    const rect = wrapper.getBoundingClientRect();
-    if (rect.width === 0 || rect.height === 0) {
-      return null;
-    }
     const x = Math.max(0, Math.min(resolution.width, ((clientX - rect.left) / rect.width) * resolution.width));
     const y = Math.max(0, Math.min(resolution.height, ((clientY - rect.top) / rect.height) * resolution.height));
-    return {
+    return { x: Math.round(x), y: Math.round(y) };
-      x: Math.round(x),
-      y: Math.round(y),
-    };
   };
 
   const buttonFromMouseEvent = (event: MouseEvent<HTMLDivElement>): DesktopMouseButton => {
@@ -224,64 +205,60 @@ export const DesktopViewer = ({
     }
   };
 
-  const withSession = (
+  const withSession = (fn: (s: DesktopStreamSession) => void) => {
-    callback: (session: NonNullable<ReturnType<DesktopViewerClient["connectDesktopStream"]>>) => void,
+    const s = sessionRef.current;
-  ) => {
+    if (s) fn(s);
-    const session = sessionRef.current;
-    if (session) {
-      callback(session);
-    }
   };
 
   return (
     <div className={className} style={{ ...shellStyle, ...style }}>
-      <div style={statusBarStyle}>
+      {showStatusBar && (
-        <span style={{ color: getStatusColor(connectionState) }}>{statusMessage}</span>
+        <div style={statusBarStyle}>
-        <span style={hintStyle}>
+          <span style={{ color: getStatusColor(connectionState) }}>{statusMessage}</span>
-          {resolution ? `${resolution.width}×${resolution.height}` : "Awaiting frames"}
+          <span style={hintStyle}>{resolution ? `${resolution.width}\u00d7${resolution.height}` : "Awaiting stream"}</span>
-        </span>
+        </div>
-      </div>
+      )}
       <div
         ref={wrapperRef}
-        role="button"
+        role="application"
-        tabIndex={0}
+        tabIndex={tabIndex}
-        style={{ ...viewportStyle, height }}
+        style={viewportStyle}
         onMouseMove={(event) => {
           const point = scalePoint(event.clientX, event.clientY);
-          if (!point) {
+          if (!point) return;
-            return;
+          withSession((s) => s.moveMouse(point.x, point.y));
-          }
-          withSession((session) => session.moveMouse(point.x, point.y));
         }}
         onMouseDown={(event) => {
           event.preventDefault();
+          // Ensure keyboard focus stays on the viewport when clicking.
+          wrapperRef.current?.focus();
           const point = scalePoint(event.clientX, event.clientY);
-          withSession((session) =>
+          if (!point) return;
-            session.mouseDown(buttonFromMouseEvent(event), point?.x, point?.y),
+          withSession((s) => s.mouseDown(buttonFromMouseEvent(event), point.x, point.y));
-          );
         }}
         onMouseUp={(event) => {
           const point = scalePoint(event.clientX, event.clientY);
-          withSession((session) => session.mouseUp(buttonFromMouseEvent(event), point?.x, point?.y));
+          if (!point) return;
+          withSession((s) => s.mouseUp(buttonFromMouseEvent(event), point.x, point.y));
         }}
         onWheel={(event: WheelEvent<HTMLDivElement>) => {
           event.preventDefault();
           const point = scalePoint(event.clientX, event.clientY);
-          if (!point) {
+          if (!point) return;
-            return;
+          withSession((s) => s.scroll(point.x, point.y, Math.round(event.deltaX), Math.round(event.deltaY)));
-          }
-          withSession((session) => session.scroll(point.x, point.y, Math.round(event.deltaX), Math.round(event.deltaY)));
         }}
         onKeyDown={(event) => {
-          withSession((session) => session.keyDown(event.key));
+          event.preventDefault();
+          event.stopPropagation();
+          withSession((s) => s.keyDown(event.key));
         }}
         onKeyUp={(event) => {
-          withSession((session) => session.keyUp(event.key));
+          event.stopPropagation();
+          withSession((s) => s.keyUp(event.key));
         }}
+        onContextMenu={(event) => event.preventDefault()}
       >
-        {frameUrl ? (
+        <video ref={videoRef} autoPlay playsInline muted style={videoBaseStyle} />
-          <img alt="Desktop stream" draggable={false} src={frameUrl} style={{ ...imageBaseStyle, ...imageStyle }} />
-        ) : null}
       </div>
     </div>
   );

sdks/typescript/src/client.ts

@@ -23,10 +23,7 @@ import {
   type SetSessionModeRequest,
 } from "acp-http-client";
 import type { SandboxAgentSpawnHandle, SandboxAgentSpawnOptions } from "./spawn.ts";
-import {
+import { DesktopStreamSession, type DesktopStreamConnectOptions } from "./desktop-stream.ts";
-  DesktopStreamSession,
-  type DesktopStreamConnectOptions,
-} from "./desktop-stream.ts";
 import {
   type AcpServerListResponse,
   type AgentInfo,
@@ -1530,9 +1527,7 @@ export class SandboxAgent {
     return this.requestJson("GET", `${API_PREFIX}/desktop/windows`);
   }
 
-  async startDesktopRecording(
+  async startDesktopRecording(request: DesktopRecordingStartRequest = {}): Promise<DesktopRecordingInfo> {
-    request: DesktopRecordingStartRequest = {},
-  ): Promise<DesktopRecordingInfo> {
     return this.requestJson("POST", `${API_PREFIX}/desktop/recording/start`, {
       body: request,
     });
@@ -1551,13 +1546,9 @@ export class SandboxAgent {
   }
 
   async downloadDesktopRecording(id: string): Promise<Uint8Array> {
-    const response = await this.requestRaw(
+    const response = await this.requestRaw("GET", `${API_PREFIX}/desktop/recordings/${encodeURIComponent(id)}/download`, {
-      "GET",
+      accept: "video/mp4",
-      `${API_PREFIX}/desktop/recordings/${encodeURIComponent(id)}/download`,
+    });
-      {
-        accept: "video/mp4",
-      },
-    );
     const buffer = await response.arrayBuffer();
     return new Uint8Array(buffer);
   }
@@ -1799,7 +1790,7 @@ export class SandboxAgent {
 
   buildDesktopStreamWebSocketUrl(options: ProcessTerminalWebSocketUrlOptions = {}): string {
     return toWebSocketUrl(
-      this.buildUrl(`${API_PREFIX}/desktop/stream/ws`, {
+      this.buildUrl(`${API_PREFIX}/desktop/stream/signaling`, {
         access_token: options.accessToken ?? this.token,
       }),
     );
@@ -1820,7 +1811,7 @@ export class SandboxAgent {
   }
 
   connectDesktopStream(options: DesktopStreamSessionOptions = {}): DesktopStreamSession {
-    return new DesktopStreamSession(this.connectDesktopStreamWebSocket(options));
+    return new DesktopStreamSession(this.connectDesktopStreamWebSocket(options), options);
   }
 
   private async getLiveConnection(agent: string): Promise<LiveAcpConnection> {

sdks/typescript/src/desktop-stream.ts

@@ -1,6 +1,5 @@
 import type { DesktopMouseButton } from "./types.ts";
 
-const WS_READY_STATE_CONNECTING = 0;
 const WS_READY_STATE_OPEN = 1;
 const WS_READY_STATE_CLOSED = 3;
 
@@ -21,63 +20,140 @@ export interface DesktopStreamConnectOptions {
   accessToken?: string;
   WebSocket?: typeof WebSocket;
   protocols?: string | string[];
+  RTCPeerConnection?: typeof RTCPeerConnection;
+  rtcConfig?: RTCConfiguration;
 }
 
-type DesktopStreamClientFrame =
+/**
-  | {
+ * Data channel binary input protocol (Big Endian).
-      type: "moveMouse";
+ *
-      x: number;
+ * Byte 0: opcode
-      y: number;
+ *   0x01 = mouse_move    (bytes 1-2: u16 BE x, bytes 3-4: u16 BE y)
-    }
+ *   0x02 = mouse_down    (byte 1: u8 button)
-  | {
+ *   0x03 = mouse_up      (byte 1: u8 button)
-      type: "mouseDown" | "mouseUp";
+ *   0x04 = mouse_scroll  (bytes 1-2: i16 BE dx, bytes 3-4: i16 BE dy)
-      x?: number;
+ *   0x05 = key_down      (bytes 1-4: u32 BE keysym)
-      y?: number;
+ *   0x06 = key_up        (bytes 1-4: u32 BE keysym)
-      button?: DesktopMouseButton;
+ */
-    }
+const OP_MOUSE_MOVE = 0x01;
-  | {
+const OP_MOUSE_DOWN = 0x02;
-      type: "scroll";
+const OP_MOUSE_UP = 0x03;
-      x: number;
+const OP_MOUSE_SCROLL = 0x04;
-      y: number;
+const OP_KEY_DOWN = 0x05;
-      deltaX?: number;
+const OP_KEY_UP = 0x06;
-      deltaY?: number;
+
-    }
+function mouseButtonToX11(button?: DesktopMouseButton): number {
-  | {
+  switch (button) {
-      type: "keyDown" | "keyUp";
+    case "middle":
-      key: string;
+      return 2;
-    }
+    case "right":
-  | {
+      return 3;
-      type: "close";
+    default:
-    };
+      return 1;
+  }
+}
+
+function keyToX11Keysym(key: string): number {
+  if (key.length === 1) {
+    const cp = key.charCodeAt(0);
+    if (cp >= 0x20 && cp <= 0x7e) return cp;
+    return 0x01000000 + cp;
+  }
+
+  const map: Record<string, number> = {
+    Backspace: 0xff08,
+    Tab: 0xff09,
+    Return: 0xff0d,
+    Enter: 0xff0d,
+    Escape: 0xff1b,
+    Delete: 0xffff,
+    Home: 0xff50,
+    Left: 0xff51,
+    ArrowLeft: 0xff51,
+    Up: 0xff52,
+    ArrowUp: 0xff52,
+    Right: 0xff53,
+    ArrowRight: 0xff53,
+    Down: 0xff54,
+    ArrowDown: 0xff54,
+    PageUp: 0xff55,
+    PageDown: 0xff56,
+    End: 0xff57,
+    Insert: 0xff63,
+    F1: 0xffbe,
+    F2: 0xffbf,
+    F3: 0xffc0,
+    F4: 0xffc1,
+    F5: 0xffc2,
+    F6: 0xffc3,
+    F7: 0xffc4,
+    F8: 0xffc5,
+    F9: 0xffc6,
+    F10: 0xffc7,
+    F11: 0xffc8,
+    F12: 0xffc9,
+    Shift: 0xffe1,
+    ShiftLeft: 0xffe1,
+    ShiftRight: 0xffe2,
+    Control: 0xffe3,
+    ControlLeft: 0xffe3,
+    ControlRight: 0xffe4,
+    Alt: 0xffe9,
+    AltLeft: 0xffe9,
+    AltRight: 0xffea,
+    Meta: 0xffeb,
+    MetaLeft: 0xffeb,
+    MetaRight: 0xffec,
+    CapsLock: 0xffe5,
+    NumLock: 0xff7f,
+    ScrollLock: 0xff14,
+    " ": 0x0020,
+    Space: 0x0020,
+  };
+
+  return map[key] ?? 0;
+}
 
 export class DesktopStreamSession {
   readonly socket: WebSocket;
   readonly closed: Promise<void>;
 
+  private pc: RTCPeerConnection | null = null;
+  private dataChannel: RTCDataChannel | null = null;
+  private mediaStream: MediaStream | null = null;
+  private connected = false;
+  private pendingCandidates: Record<string, unknown>[] = [];
+  private cachedReadyStatus: DesktopStreamReadyStatus | null = null;
+
   private readonly readyListeners = new Set<(status: DesktopStreamReadyStatus) => void>();
-  private readonly frameListeners = new Set<(frame: Uint8Array) => void>();
+  private readonly trackListeners = new Set<(stream: MediaStream) => void>();
+  private readonly connectListeners = new Set<() => void>();
+  private readonly disconnectListeners = new Set<() => void>();
   private readonly errorListeners = new Set<(error: DesktopStreamErrorStatus | Error) => void>();
-  private readonly closeListeners = new Set<() => void>();
 
-  private closeSignalSent = false;
   private closedResolve!: () => void;
+  private readonly PeerConnection: typeof RTCPeerConnection;
+  private readonly rtcConfig: RTCConfiguration;
 
-  constructor(socket: WebSocket) {
+  constructor(socket: WebSocket, options: DesktopStreamConnectOptions = {}) {
     this.socket = socket;
-    this.socket.binaryType = "arraybuffer";
+    this.PeerConnection = options.RTCPeerConnection ?? globalThis.RTCPeerConnection;
+    this.rtcConfig = options.rtcConfig ?? {};
+
     this.closed = new Promise<void>((resolve) => {
       this.closedResolve = resolve;
     });
 
     this.socket.addEventListener("message", (event) => {
-      void this.handleMessage(event.data);
+      this.handleMessage(event.data as string);
     });
     this.socket.addEventListener("error", () => {
-      this.emitError(new Error("Desktop stream websocket connection failed."));
+      this.emitError(new Error("Desktop stream signaling connection failed."));
     });
     this.socket.addEventListener("close", () => {
+      this.teardownPeerConnection();
       this.closedResolve();
-      for (const listener of this.closeListeners) {
+      for (const listener of this.disconnectListeners) {
         listener();
       }
     });
@@ -85,15 +161,35 @@ export class DesktopStreamSession {
 
   onReady(listener: (status: DesktopStreamReadyStatus) => void): () => void {
     this.readyListeners.add(listener);
+    if (this.cachedReadyStatus) {
+      listener(this.cachedReadyStatus);
+    }
     return () => {
       this.readyListeners.delete(listener);
     };
   }
 
-  onFrame(listener: (frame: Uint8Array) => void): () => void {
+  onTrack(listener: (stream: MediaStream) => void): () => void {
-    this.frameListeners.add(listener);
+    this.trackListeners.add(listener);
+    if (this.mediaStream) {
+      listener(this.mediaStream);
+    }
     return () => {
-      this.frameListeners.delete(listener);
+      this.trackListeners.delete(listener);
+    };
+  }
+
+  onConnect(listener: () => void): () => void {
+    this.connectListeners.add(listener);
+    return () => {
+      this.connectListeners.delete(listener);
+    };
+  }
+
+  onDisconnect(listener: () => void): () => void {
+    this.disconnectListeners.add(listener);
+    return () => {
+      this.disconnectListeners.delete(listener);
     };
   }
 
@@ -104,97 +200,313 @@ export class DesktopStreamSession {
     };
   }
 
+  /** @deprecated Use onDisconnect instead. */
   onClose(listener: () => void): () => void {
-    this.closeListeners.add(listener);
+    return this.onDisconnect(listener);
-    return () => {
+  }
-      this.closeListeners.delete(listener);
+
-    };
+  /** @deprecated No longer emits JPEG frames. Use onTrack for WebRTC media. */
+  onFrame(_listener: (frame: Uint8Array) => void): () => void {
+    return () => {};
+  }
+
+  getMediaStream(): MediaStream | null {
+    return this.mediaStream;
   }
 
   moveMouse(x: number, y: number): void {
-    this.sendFrame({ type: "moveMouse", x, y });
+    if (this.dataChannel?.readyState === "open") {
+      const buf = new ArrayBuffer(5);
+      const view = new DataView(buf);
+      view.setUint8(0, OP_MOUSE_MOVE);
+      view.setUint16(1, x, false);
+      view.setUint16(3, y, false);
+      this.dataChannel.send(buf);
+    } else {
+      this.sendSignaling("moveMouse", { x, y });
+    }
   }
 
   mouseDown(button?: DesktopMouseButton, x?: number, y?: number): void {
-    this.sendFrame({ type: "mouseDown", button, x, y });
+    if (x != null && y != null) {
+      this.moveMouse(x, y);
+    }
+    if (this.dataChannel?.readyState === "open") {
+      const buf = new ArrayBuffer(2);
+      const view = new DataView(buf);
+      view.setUint8(0, OP_MOUSE_DOWN);
+      view.setUint8(1, mouseButtonToX11(button));
+      this.dataChannel.send(buf);
+    } else {
+      this.sendSignaling("mouseDown", { button, x, y });
+    }
   }
 
   mouseUp(button?: DesktopMouseButton, x?: number, y?: number): void {
-    this.sendFrame({ type: "mouseUp", button, x, y });
+    if (x != null && y != null) {
+      this.moveMouse(x, y);
+    }
+    if (this.dataChannel?.readyState === "open") {
+      const buf = new ArrayBuffer(2);
+      const view = new DataView(buf);
+      view.setUint8(0, OP_MOUSE_UP);
+      view.setUint8(1, mouseButtonToX11(button));
+      this.dataChannel.send(buf);
+    } else {
+      this.sendSignaling("mouseUp", { button, x, y });
+    }
   }
 
   scroll(x: number, y: number, deltaX?: number, deltaY?: number): void {
-    this.sendFrame({ type: "scroll", x, y, deltaX, deltaY });
+    this.moveMouse(x, y);
+    if (this.dataChannel?.readyState === "open") {
+      const buf = new ArrayBuffer(5);
+      const view = new DataView(buf);
+      view.setUint8(0, OP_MOUSE_SCROLL);
+      view.setInt16(1, deltaX ?? 0, false);
+      view.setInt16(3, deltaY ?? 0, false);
+      this.dataChannel.send(buf);
+    } else {
+      this.sendSignaling("scroll", { x, y, deltaX, deltaY });
+    }
   }
 
   keyDown(key: string): void {
-    this.sendFrame({ type: "keyDown", key });
+    const keysym = keyToX11Keysym(key);
+    if (keysym === 0) return;
+    if (this.dataChannel?.readyState === "open") {
+      const buf = new ArrayBuffer(5);
+      const view = new DataView(buf);
+      view.setUint8(0, OP_KEY_DOWN);
+      view.setUint32(1, keysym, false);
+      this.dataChannel.send(buf);
+    } else {
+      this.sendSignaling("keyDown", { key });
+    }
   }
 
   keyUp(key: string): void {
-    this.sendFrame({ type: "keyUp", key });
+    const keysym = keyToX11Keysym(key);
+    if (keysym === 0) return;
+    if (this.dataChannel?.readyState === "open") {
+      const buf = new ArrayBuffer(5);
+      const view = new DataView(buf);
+      view.setUint8(0, OP_KEY_UP);
+      view.setUint32(1, keysym, false);
+      this.dataChannel.send(buf);
+    } else {
+      this.sendSignaling("keyUp", { key });
+    }
   }
 
   close(): void {
-    if (this.socket.readyState === WS_READY_STATE_CONNECTING) {
+    this.teardownPeerConnection();
-      this.socket.addEventListener(
-        "open",
-        () => {
-          this.close();
-        },
-        { once: true },
-      );
-      return;
-    }
-
-    if (this.socket.readyState === WS_READY_STATE_OPEN) {
-      if (!this.closeSignalSent) {
-        this.closeSignalSent = true;
-        this.sendFrame({ type: "close" });
-      }
-      this.socket.close();
-      return;
-    }
-
     if (this.socket.readyState !== WS_READY_STATE_CLOSED) {
       this.socket.close();
     }
   }
 
-  private async handleMessage(data: unknown): Promise<void> {
+  private handleMessage(data: string): void {
+    let msg: Record<string, unknown>;
     try {
-      if (typeof data === "string") {
+      msg = JSON.parse(data) as Record<string, unknown>;
-        const frame = parseStatusFrame(data);
+    } catch {
-        if (!frame) {
+      return;
-          this.emitError(new Error("Received invalid desktop stream control frame."));
+    }
-          return;
-        }
 
-        if (frame.type === "ready") {
+    const type = (msg.type as string) ?? "";
-          for (const listener of this.readyListeners) {
-            listener(frame);
-          }
-          return;
-        }
 
-        this.emitError(frame);
+    switch (type) {
-        return;
+      case "ready": {
+        const status: DesktopStreamReadyStatus = {
+          type: "ready",
+          width: Number(msg.width) || 0,
+          height: Number(msg.height) || 0,
+        };
+        this.cachedReadyStatus = status;
+        for (const listener of this.readyListeners) {
+          listener(status);
+        }
+        break;
       }
 
-      const bytes = await decodeBinaryFrame(data);
+      case "offer": {
-      for (const listener of this.frameListeners) {
+        if (msg.sdp) {
-        listener(bytes);
+          void this.handleOffer(msg.sdp as string);
+        }
+        break;
       }
+
+      case "candidate": {
+        void this.handleCandidate(msg as unknown as RTCIceCandidateInit);
+        break;
+      }
+
+      case "error": {
+        const errorStatus: DesktopStreamErrorStatus = {
+          type: "error",
+          message: (msg.message as string) ?? "Unknown error",
+        };
+        this.emitError(errorStatus);
+        break;
+      }
+
+      default:
+        break;
+    }
+  }
+
+  private async handleOffer(sdp: string): Promise<void> {
+    try {
+      const config: RTCConfiguration = {
+        ...this.rtcConfig,
+        iceServers: this.rtcConfig.iceServers ?? [{ urls: "stun:stun.l.google.com:19302" }],
+      };
+      const pc = new this.PeerConnection(config);
+      this.pc = pc;
+
+      pc.ontrack = (event) => {
+        const stream = event.streams[0] ?? new MediaStream([event.track]);
+        this.mediaStream = stream;
+        for (const listener of this.trackListeners) {
+          listener(stream);
+        }
+      };
+
+      pc.onicecandidate = (event) => {
+        if (event.candidate) {
+          this.sendJson({
+            type: "candidate",
+            candidate: event.candidate.candidate,
+            sdpMLineIndex: event.candidate.sdpMLineIndex,
+            sdpMid: event.candidate.sdpMid,
+          });
+        }
+      };
+
+      pc.onconnectionstatechange = () => {
+        switch (pc.connectionState) {
+          case "connected":
+            if (!this.connected) {
+              this.connected = true;
+              for (const listener of this.connectListeners) {
+                listener();
+              }
+            }
+            break;
+          case "closed":
+          case "failed":
+            this.emitError(new Error(`WebRTC connection ${pc.connectionState}.`));
+            break;
+        }
+      };
+
+      pc.oniceconnectionstatechange = () => {
+        switch (pc.iceConnectionState) {
+          case "connected":
+            if (!this.connected) {
+              this.connected = true;
+              for (const listener of this.connectListeners) {
+                listener();
+              }
+            }
+            break;
+          case "closed":
+          case "failed":
+            this.emitError(new Error(`WebRTC ICE ${pc.iceConnectionState}.`));
+            break;
+        }
+      };
+
+      // Server creates the data channel; client receives it.
+      pc.ondatachannel = (event) => {
+        this.dataChannel = event.channel;
+        this.dataChannel.binaryType = "arraybuffer";
+        this.dataChannel.onerror = () => {
+          this.emitError(new Error("WebRTC data channel error."));
+        };
+        this.dataChannel.onclose = () => {
+          this.dataChannel = null;
+        };
+      };
+
+      await pc.setRemoteDescription({ type: "offer", sdp });
+
+      // Flush any ICE candidates that arrived before the PC was ready.
+      for (const pending of this.pendingCandidates) {
+        try {
+          await pc.addIceCandidate(pending as unknown as RTCIceCandidateInit);
+        } catch {
+          // ignore stale candidates
+        }
+      }
+      this.pendingCandidates = [];
+
+      const answer = await pc.createAnswer();
+      await pc.setLocalDescription(answer);
+
+      this.sendJson({ type: "answer", sdp: answer.sdp });
     } catch (error) {
       this.emitError(error instanceof Error ? error : new Error(String(error)));
     }
   }
 
-  private sendFrame(frame: DesktopStreamClientFrame): void {
+  private async handleCandidate(candidate: RTCIceCandidateInit): Promise<void> {
-    if (this.socket.readyState !== WS_READY_STATE_OPEN) {
+    if (!this.pc) {
+      this.pendingCandidates.push(candidate as unknown as Record<string, unknown>);
       return;
     }
-    this.socket.send(JSON.stringify(frame));
+    try {
+      await this.pc.addIceCandidate(candidate);
+    } catch (error) {
+      this.emitError(error instanceof Error ? error : new Error(String(error)));
+    }
+  }
+
+  /** Send a JSON message to the server. */
+  private sendJson(msg: Record<string, unknown>): void {
+    if (this.socket.readyState !== WS_READY_STATE_OPEN) return;
+    this.socket.send(JSON.stringify(msg));
+  }
+
+  /** Send a typed input message over the signaling WebSocket as fallback. */
+  private sendSignaling(type: string, data: Record<string, unknown>): void {
+    this.sendJson({ type, ...data });
+  }
+
+  /** Tear down the peer connection, nullifying handlers first to prevent stale
+   *  callbacks. */
+  private teardownPeerConnection(): void {
+    if (this.dataChannel) {
+      this.dataChannel.onerror = null;
+      this.dataChannel.onmessage = null;
+      this.dataChannel.onopen = null;
+      this.dataChannel.onclose = null;
+      try {
+        this.dataChannel.close();
+      } catch {
+        /* ignore */
+      }
+      this.dataChannel = null;
+    }
+    if (this.pc) {
+      this.pc.onicecandidate = null;
+      this.pc.onicecandidateerror = null;
+      this.pc.onconnectionstatechange = null;
+      this.pc.oniceconnectionstatechange = null;
+      this.pc.onsignalingstatechange = null;
+      this.pc.onnegotiationneeded = null;
+      this.pc.ontrack = null;
+      this.pc.ondatachannel = null;
+      try {
+        this.pc.close();
+      } catch {
+        /* ignore */
+      }
+      this.pc = null;
+    }
+    this.mediaStream = null;
+    this.connected = false;
   }
 
   private emitError(error: DesktopStreamErrorStatus | Error): void {
@@ -203,34 +515,3 @@ export class DesktopStreamSession {
     }
   }
 }
-
-function parseStatusFrame(payload: string): DesktopStreamStatusMessage | null {
-  const value = JSON.parse(payload) as Record<string, unknown>;
-  if (value.type === "ready" && typeof value.width === "number" && typeof value.height === "number") {
-    return {
-      type: "ready",
-      width: value.width,
-      height: value.height,
-    };
-  }
-  if (value.type === "error" && typeof value.message === "string") {
-    return {
-      type: "error",
-      message: value.message,
-    };
-  }
-  return null;
-}
-
-async function decodeBinaryFrame(data: unknown): Promise<Uint8Array> {
-  if (data instanceof ArrayBuffer) {
-    return new Uint8Array(data);
-  }
-  if (ArrayBuffer.isView(data)) {
-    return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
-  }
-  if (typeof Blob !== "undefined" && data instanceof Blob) {
-    return new Uint8Array(await data.arrayBuffer());
-  }
-  throw new Error("Unsupported desktop stream binary frame type.");
-}

sdks/typescript/src/generated/openapi.ts

@@ -3,7 +3,6 @@
  * Do not make direct changes to the file.
  */
 
-
 export interface paths {
   "/v1/acp": {
     get: operations["get_v1_acp_servers"];
@@ -225,9 +224,10 @@ export interface paths {
   };
   "/v1/desktop/stream/ws": {
     /**
-     * Open a desktop websocket streaming session.
+     * Open a desktop WebRTC signaling session.
-     * @description Upgrades the connection to a websocket that streams JPEG desktop frames and
+     * @description Upgrades the connection to a WebSocket used for WebRTC signaling between
-     * accepts mouse and keyboard control frames.
+     * the browser client and the desktop streaming process. Also accepts mouse
+     * and keyboard input frames as a fallback transport.
      */
     get: operations["get_v1_desktop_stream_ws"];
   };
@@ -633,7 +633,23 @@ export interface components {
       windows: components["schemas"]["DesktopWindowInfo"][];
     };
     /** @enum {string} */
-    ErrorType: "invalid_request" | "conflict" | "unsupported_agent" | "agent_not_installed" | "install_failed" | "agent_process_exited" | "token_invalid" | "permission_denied" | "not_acceptable" | "unsupported_media_type" | "not_found" | "session_not_found" | "session_already_exists" | "mode_not_supported" | "stream_error" | "timeout";
+    ErrorType:
+      | "invalid_request"
+      | "conflict"
+      | "unsupported_agent"
+      | "agent_not_installed"
+      | "install_failed"
+      | "agent_process_exited"
+      | "token_invalid"
+      | "permission_denied"
+      | "not_acceptable"
+      | "unsupported_media_type"
+      | "not_found"
+      | "session_not_found"
+      | "session_already_exists"
+      | "mode_not_supported"
+      | "stream_error"
+      | "timeout";
     FsActionResponse: {
       path: string;
     };
@@ -692,35 +708,37 @@ export interface components {
       directory: string;
       mcpName: string;
     };
-    McpServerConfig: ({
+    McpServerConfig:
-      args?: string[];
+      | {
-      command: string;
+          args?: string[];
-      cwd?: string | null;
+          command: string;
-      enabled?: boolean | null;
+          cwd?: string | null;
-      env?: {
+          enabled?: boolean | null;
-        [key: string]: string;
+          env?: {
-      } | null;
+            [key: string]: string;
-      /** Format: int64 */
+          } | null;
-      timeoutMs?: number | null;
+          /** Format: int64 */
-      /** @enum {string} */
+          timeoutMs?: number | null;
-      type: "local";
+          /** @enum {string} */
-    }) | ({
+          type: "local";
-      bearerTokenEnvVar?: string | null;
+        }
-      enabled?: boolean | null;
+      | {
-      envHeaders?: {
+          bearerTokenEnvVar?: string | null;
-        [key: string]: string;
+          enabled?: boolean | null;
-      } | null;
+          envHeaders?: {
-      headers?: {
+            [key: string]: string;
-        [key: string]: string;
+          } | null;
-      } | null;
+          headers?: {
-      oauth?: Record<string, unknown> | null | null;
+            [key: string]: string;
-      /** Format: int64 */
+          } | null;
-      timeoutMs?: number | null;
+          oauth?: Record<string, unknown> | null | null;
-      transport?: string | null;
+          /** Format: int64 */
-      /** @enum {string} */
+          timeoutMs?: number | null;
-      type: "remote";
+          transport?: string | null;
-      url: string;
+          /** @enum {string} */
-    });
+          type: "remote";
+          url: string;
+        };
     ProblemDetails: {
       detail?: string | null;
       instance?: string | null;
@@ -880,7 +898,6 @@ export type $defs = Record<string, never>;
 export type external = Record<string, never>;
 
 export interface operations {
-
   get_v1_acp_servers: {
     responses: {
       /** @description Active ACP server instances */
@@ -2002,9 +2019,10 @@ export interface operations {
     };
   };
   /**
-   * Open a desktop websocket streaming session.
+   * Open a desktop WebRTC signaling session.
-   * @description Upgrades the connection to a websocket that streams JPEG desktop frames and
+   * @description Upgrades the connection to a WebSocket used for WebRTC signaling between
-   * accepts mouse and keyboard control frames.
+   * the browser client and the desktop streaming process. Also accepts mouse
+   * and keyboard input frames as a fallback transport.
    */
   get_v1_desktop_stream_ws: {
     parameters: {

server/compose.dev.yaml

@@ -0,0 +1,40 @@
+name: sandbox-agent-dev
+
+services:
+  backend:
+    build:
+      context: ..
+      dockerfile: docker/test-agent/Dockerfile
+    image: sandbox-agent-dev
+    command: ["server", "--host", "0.0.0.0", "--port", "3000", "--no-token"]
+    environment:
+      RUST_LOG: "${RUST_LOG:-info}"
+    ports:
+      - "2468:3000"
+
+  frontend:
+    build:
+      context: ..
+      dockerfile: server/docker/frontend.dev.Dockerfile
+    working_dir: /app
+    depends_on:
+      - backend
+    environment:
+      SANDBOX_AGENT_URL: "http://backend:3000"
+    ports:
+      - "5173:5173"
+    volumes:
+      - "..:/app"
+      # Keep Linux-native node_modules inside the container.
+      - "sa_root_node_modules:/app/node_modules"
+      - "sa_inspector_node_modules:/app/frontend/packages/inspector/node_modules"
+      - "sa_react_node_modules:/app/sdks/react/node_modules"
+      - "sa_typescript_node_modules:/app/sdks/typescript/node_modules"
+      - "sa_pnpm_store:/root/.local/share/pnpm/store"
+
+volumes:
+  sa_root_node_modules: {}
+  sa_inspector_node_modules: {}
+  sa_react_node_modules: {}
+  sa_typescript_node_modules: {}
+  sa_pnpm_store: {}

server/docker/frontend.dev.Dockerfile

@@ -0,0 +1,5 @@
+FROM node:22-bookworm-slim
+RUN npm install -g pnpm
+WORKDIR /app
+EXPOSE 5173
+CMD ["sh", "-c", "pnpm install && cd frontend/packages/inspector && npx vite --host 0.0.0.0"]

server/packages/sandbox-agent/Cargo.toml

@@ -42,6 +42,9 @@ toml_edit.workspace = true
 tar.workspace = true
 zip.workspace = true
 tempfile = { workspace = true, optional = true }
+gstreamer = { version = "0.23", optional = true }
+gstreamer-sdp = { version = "0.23", optional = true }
+gstreamer-webrtc = { version = "0.23", optional = true }
 
 [target.'cfg(unix)'.dependencies]
 libc = "0.2"
@@ -59,3 +62,4 @@ tokio-tungstenite = "0.24"
 
 [features]
 test-utils = ["tempfile"]
+desktop-gstreamer = ["gstreamer", "gstreamer-sdp", "gstreamer-webrtc"]

server/packages/sandbox-agent/src/desktop_gstreamer.rs

@@ -0,0 +1,246 @@
+/// GStreamer WebRTC pipeline for desktop streaming.
+///
+/// Creates a pipeline that captures the X11 display via `ximagesrc`, encodes to
+/// VP8, and streams over WebRTC using `webrtcbin`. Signaling (SDP offer/answer,
+/// ICE candidate exchange) is handled via channels that the caller bridges to
+/// the client WebSocket.
+#[cfg(feature = "desktop-gstreamer")]
+pub mod pipeline {
+    use gstreamer as gst;
+    use gstreamer::prelude::*;
+    use gstreamer_sdp as gst_sdp;
+    use gstreamer_webrtc as gst_webrtc;
+    use tokio::sync::mpsc;
+
+    /// Messages sent from the GStreamer pipeline to the WebSocket handler.
+    #[derive(Debug)]
+    pub enum PipelineEvent {
+        /// SDP offer generated by webrtcbin.
+        Offer(String),
+        /// ICE candidate produced by webrtcbin.
+        IceCandidate {
+            candidate: String,
+            sdp_m_line_index: u32,
+        },
+    }
+
+    /// Messages sent from the WebSocket handler to the GStreamer pipeline.
+    #[derive(Debug)]
+    pub enum SignalingCommand {
+        /// SDP answer from the client.
+        Answer(String),
+        /// ICE candidate from the client.
+        IceCandidate {
+            candidate: String,
+            sdp_m_line_index: u32,
+        },
+    }
+
+    pub struct GStreamerPipeline {
+        pipeline: gst::Pipeline,
+        cmd_tx: mpsc::UnboundedSender<SignalingCommand>,
+    }
+
+    impl GStreamerPipeline {
+        /// Create and start a new GStreamer WebRTC pipeline for the given display.
+        ///
+        /// Returns the pipeline handle and a receiver for pipeline events (offers,
+        /// ICE candidates) that should be forwarded to the client.
+        pub fn new(
+            display: &str,
+        ) -> Result<(Self, mpsc::UnboundedReceiver<PipelineEvent>), String> {
+            gst::init().map_err(|e| {
+                format!(
+                    "Desktop streaming requires GStreamer. Install it with: \
+                     sandbox-agent desktop install\n\
+                     Error: {e}"
+                )
+            })?;
+
+            let pipeline_str = format!(
+                "ximagesrc display-name={display} use-damage=true show-pointer=true \
+                 ! video/x-raw,framerate=30/1 \
+                 ! videorate \
+                 ! videoconvert \
+                 ! queue max-size-buffers=1 leaky=downstream \
+                 ! vp8enc deadline=1 target-bitrate=3000000 cpu-used=16 threads=4 \
+                   keyframe-max-dist=60 end-usage=cbr buffer-size=500 buffer-initial-size=300 \
+                   error-resilient=partitions \
+                 ! rtpvp8pay picture-id-mode=15bit \
+                 ! queue max-size-buffers=1 leaky=downstream \
+                 ! application/x-rtp,media=video,encoding-name=VP8,payload=96 \
+                 ! webrtcbin name=wb bundle-policy=max-bundle"
+            );
+
+            let pipeline = gst::parse::launch(&pipeline_str)
+                .map_err(|e| format!("failed to create GStreamer pipeline: {e}"))?
+                .downcast::<gst::Pipeline>()
+                .map_err(|_| "pipeline is not a GstPipeline".to_string())?;
+
+            let webrtcbin = pipeline
+                .by_name("wb")
+                .ok_or_else(|| "webrtcbin element not found in pipeline".to_string())?;
+
+            // Configure STUN for ICE connectivity (used for server-reflexive
+            // candidates when behind NAT).
+            webrtcbin.set_property_from_str("stun-server", "stun://stun.l.google.com:19302");
+
+            // Restrict the UDP port range so Docker port forwarding works.
+            // The ice-agent is a GstWebRTCICE which wraps a NiceAgent.
+            let ice_agent: gst::glib::Object = webrtcbin.property("ice-agent");
+            // GstWebRTCNice has a "min-rtp-port" and "max-rtp-port" property
+            // in newer versions, but on GStreamer 1.22 we need to access the
+            // underlying NiceAgent via the "agent" property.
+            if ice_agent.has_property("min-rtp-port", None) {
+                ice_agent.set_property("min-rtp-port", 30000u32);
+                ice_agent.set_property("max-rtp-port", 30100u32);
+            } else if ice_agent.has_property("agent", None) {
+                let nice_agent: gst::glib::Object = ice_agent.property("agent");
+                nice_agent.set_property("max-port", 30100u32);
+                nice_agent.set_property("min-port", 30000u32);
+            }
+
+            // Channel for pipeline -> WS handler events.
+            let (event_tx, event_rx) = mpsc::unbounded_channel::<PipelineEvent>();
+
+            // Channel for WS handler -> pipeline commands.
+            let (cmd_tx, mut cmd_rx) = mpsc::unbounded_channel::<SignalingCommand>();
+
+            // Note: Data channel for input will be created once we establish
+            // the WebRTC connection. Input falls back to the WS transport.
+
+            // When webrtcbin needs to negotiate, create an offer.
+            let wb_clone = webrtcbin.clone();
+            let event_tx_offer = event_tx.clone();
+            webrtcbin.connect("on-negotiation-needed", false, move |_| {
+                let wb_offer = wb_clone.clone();
+                let wb_create = wb_clone.clone();
+                let tx = event_tx_offer.clone();
+                let promise = gst::Promise::with_change_func(move |reply| {
+                    let reply = match reply {
+                        Ok(Some(reply)) => reply,
+                        _ => return,
+                    };
+                    let offer = match reply.value("offer") {
+                        Ok(offer) => offer,
+                        Err(_) => return,
+                    };
+                    let offer = offer
+                        .get::<gst_webrtc::WebRTCSessionDescription>()
+                        .expect("offer is WebRTCSessionDescription");
+                    wb_offer.emit_by_name::<()>(
+                        "set-local-description",
+                        &[&offer, &None::<gst::Promise>],
+                    );
+                    if let Ok(sdp_text) = offer.sdp().as_text() {
+                        let _ = tx.send(PipelineEvent::Offer(sdp_text.to_string()));
+                    }
+                });
+                wb_create.emit_by_name::<()>("create-offer", &[&None::<gst::Structure>, &promise]);
+                None
+            });
+
+            // When webrtcbin produces an ICE candidate, send it to client.
+            // We rewrite host candidates to use 127.0.0.1 so the browser can
+            // reach the server when running inside Docker.
+            let event_tx_ice = event_tx;
+            webrtcbin.connect("on-ice-candidate", false, move |values| {
+                let sdp_m_line_index = values[1].get::<u32>().expect("m-line index is u32");
+                let candidate = values[2].get::<String>().expect("candidate is String");
+
+                // Only forward UDP host candidates, rewritten to 127.0.0.1.
+                // Skip TCP candidates (browsers rarely use TCP for WebRTC media)
+                // and server-reflexive candidates (STUN responses with public IPs).
+                if candidate.contains("UDP") && candidate.contains("typ host") {
+                    // Replace the Docker-internal IP with 127.0.0.1 so the
+                    // browser on the host can connect.
+                    let rewritten = rewrite_candidate_ip(&candidate, "127.0.0.1");
+                    let _ = event_tx_ice.send(PipelineEvent::IceCandidate {
+                        candidate: rewritten,
+                        sdp_m_line_index,
+                    });
+                }
+
+                None
+            });
+
+            // Start the pipeline.
+            pipeline
+                .set_state(gst::State::Playing)
+                .map_err(|e| format!("failed to start GStreamer pipeline: {e}"))?;
+
+            // Spawn a thread to process signaling commands from the WS handler.
+            let wb_cmd = webrtcbin.clone();
+            std::thread::spawn(move || {
+                while let Some(cmd) = cmd_rx.blocking_recv() {
+                    match cmd {
+                        SignalingCommand::Answer(sdp_str) => {
+                            let sdp = match gst_sdp::SDPMessage::parse_buffer(sdp_str.as_bytes()) {
+                                Ok(sdp) => sdp,
+                                Err(e) => {
+                                    tracing::warn!(error = ?e, "failed to parse SDP answer");
+                                    continue;
+                                }
+                            };
+                            let answer = gst_webrtc::WebRTCSessionDescription::new(
+                                gst_webrtc::WebRTCSDPType::Answer,
+                                sdp,
+                            );
+                            wb_cmd.emit_by_name::<()>(
+                                "set-remote-description",
+                                &[&answer, &None::<gst::Promise>],
+                            );
+                        }
+                        SignalingCommand::IceCandidate {
+                            candidate,
+                            sdp_m_line_index,
+                        } => {
+                            wb_cmd.emit_by_name::<()>(
+                                "add-ice-candidate",
+                                &[&sdp_m_line_index, &candidate],
+                            );
+                        }
+                    }
+                }
+            });
+
+            Ok((Self { pipeline, cmd_tx }, event_rx))
+        }
+
+        /// Send a signaling command to the pipeline.
+        pub fn send_command(&self, cmd: SignalingCommand) {
+            let _ = self.cmd_tx.send(cmd);
+        }
+    }
+
+    impl Drop for GStreamerPipeline {
+        fn drop(&mut self) {
+            let _ = self.pipeline.set_state(gst::State::Null);
+        }
+    }
+
+    /// Rewrite the IP address in an ICE candidate string.
+    ///
+    /// ICE candidate format:
+    ///   candidate:1 1 UDP 2015363327 172.17.0.6 39395 typ host
+    ///
+    /// We replace the IP (field 5, 0-indexed) with the target IP.
+    fn rewrite_candidate_ip(candidate: &str, target_ip: &str) -> String {
+        let parts: Vec<&str> = candidate.splitn(6, ' ').collect();
+        if parts.len() >= 6 {
+            // parts[4] is the IP address
+            let rest_after_ip = &candidate[parts[..5].join(" ").len()..];
+            format!(
+                "{} {} {} {} {}{}",
+                parts[0], parts[1], parts[2], parts[3], target_ip, rest_after_ip
+            )
+        } else {
+            candidate.to_string()
+        }
+    }
+}
+
+/// Check if GStreamer support is compiled in.
+pub fn is_available() -> bool {
+    cfg!(feature = "desktop-gstreamer")
+}

server/packages/sandbox-agent/src/desktop_install.rs

@@ -110,6 +110,13 @@ fn desktop_packages(package_manager: DesktopPackageManager, no_fonts: bool) -> V
             "dbus-x11",
             "xauth",
             "fonts-dejavu-core",
+            "libgstreamer1.0-0",
+            "gstreamer1.0-plugins-base",
+            "gstreamer1.0-plugins-good",
+            "gstreamer1.0-plugins-bad",
+            "gstreamer1.0-plugins-ugly",
+            "gstreamer1.0-nice",
+            "gstreamer1.0-x",
         ],
         DesktopPackageManager::Dnf => vec![
             "xorg-x11-server-Xvfb",
@@ -121,6 +128,13 @@ fn desktop_packages(package_manager: DesktopPackageManager, no_fonts: bool) -> V
             "dbus-x11",
             "xauth",
             "dejavu-sans-fonts",
+            "gstreamer1",
+            "gstreamer1-plugins-base",
+            "gstreamer1-plugins-good",
+            "gstreamer1-plugins-bad-free",
+            "gstreamer1-plugins-ugly-free",
+            "gstreamer1-plugin-libnice",
+            "gstreamer1-plugins-good-extras",
         ],
         DesktopPackageManager::Apk => vec![
             "xvfb",
@@ -132,6 +146,12 @@ fn desktop_packages(package_manager: DesktopPackageManager, no_fonts: bool) -> V
             "dbus",
             "xauth",
             "ttf-dejavu",
+            "gstreamer",
+            "gst-plugins-base",
+            "gst-plugins-good",
+            "gst-plugins-bad",
+            "gst-plugins-ugly",
+            "libnice-gstreamer",
         ],
     }
     .into_iter()

server/packages/sandbox-agent/src/desktop_runtime.rs

@@ -10,20 +10,20 @@ use tokio::sync::Mutex;
 
 use sandbox_agent_error::SandboxError;
 
-use crate::desktop_recording::{DesktopRecordingContext, DesktopRecordingManager};
 use crate::desktop_errors::DesktopProblem;
 use crate::desktop_install::desktop_platform_support_message;
+use crate::desktop_recording::{DesktopRecordingContext, DesktopRecordingManager};
 use crate::desktop_streaming::DesktopStreamingManager;
 use crate::desktop_types::{
-    DesktopActionResponse, DesktopDisplayInfoResponse, DesktopErrorInfo,
+    DesktopActionResponse, DesktopDisplayInfoResponse, DesktopErrorInfo, DesktopKeyModifiers,
-    DesktopKeyModifiers, DesktopKeyboardDownRequest, DesktopKeyboardPressRequest,
+    DesktopKeyboardDownRequest, DesktopKeyboardPressRequest, DesktopKeyboardTypeRequest,
-    DesktopKeyboardTypeRequest, DesktopKeyboardUpRequest, DesktopMouseButton,
+    DesktopKeyboardUpRequest, DesktopMouseButton, DesktopMouseClickRequest,
-    DesktopMouseClickRequest, DesktopMouseDownRequest, DesktopMouseDragRequest,
+    DesktopMouseDownRequest, DesktopMouseDragRequest, DesktopMouseMoveRequest,
-    DesktopMouseMoveRequest, DesktopMousePositionResponse, DesktopMouseScrollRequest,
+    DesktopMousePositionResponse, DesktopMouseScrollRequest, DesktopMouseUpRequest,
-    DesktopMouseUpRequest, DesktopProcessInfo, DesktopRecordingInfo,
+    DesktopProcessInfo, DesktopRecordingInfo, DesktopRecordingListResponse,
-    DesktopRecordingListResponse, DesktopRecordingStartRequest, DesktopRegionScreenshotQuery,
+    DesktopRecordingStartRequest, DesktopRegionScreenshotQuery, DesktopResolution,
-    DesktopResolution, DesktopScreenshotFormat, DesktopScreenshotQuery, DesktopStartRequest,
+    DesktopScreenshotFormat, DesktopScreenshotQuery, DesktopStartRequest, DesktopState,
-    DesktopState, DesktopStatusResponse, DesktopStreamStatusResponse, DesktopWindowInfo,
+    DesktopStatusResponse, DesktopStreamStatusResponse, DesktopWindowInfo,
     DesktopWindowListResponse,
 };
 use crate::process_runtime::{
@@ -172,9 +172,9 @@ impl DesktopRuntime {
         let recording_manager =
             DesktopRecordingManager::new(process_runtime.clone(), config.state_dir.clone());
         Self {
+            streaming_manager: DesktopStreamingManager::new(),
             process_runtime,
             recording_manager,
-            streaming_manager: DesktopStreamingManager::new(),
             inner: Arc::new(Mutex::new(DesktopRuntimeStateData {
                 state: DesktopState::Inactive,
                 display_num: config.display_num,
@@ -197,7 +197,10 @@ impl DesktopRuntime {
     pub async fn status(&self) -> DesktopStatusResponse {
         let mut state = self.inner.lock().await;
         self.refresh_status_locked(&mut state).await;
-        self.snapshot_locked(&state)
+        let mut response = self.snapshot_locked(&state);
+        drop(state);
+
+        response
     }
 
     pub async fn start(
@@ -221,7 +224,10 @@ impl DesktopRuntime {
 
         self.refresh_status_locked(&mut state).await;
         if state.state == DesktopState::Active {
-            return Ok(self.snapshot_locked(&state));
+            let mut response = self.snapshot_locked(&state);
+            drop(state);
+
+            return Ok(response);
         }
 
         if !state.missing_dependencies.is_empty() {
@@ -307,7 +313,10 @@ impl DesktopRuntime {
             ),
         );
 
-        Ok(self.snapshot_locked(&state))
+        let mut response = self.snapshot_locked(&state);
+        drop(state);
+
+        Ok(response)
     }
 
     pub async fn stop(&self) -> Result<DesktopStatusResponse, DesktopProblem> {
@@ -336,7 +345,10 @@ impl DesktopRuntime {
         state.install_command = self.install_command_for(&state.missing_dependencies);
         state.environment.clear();
 
-        Ok(self.snapshot_locked(&state))
+        let mut response = self.snapshot_locked(&state);
+        drop(state);
+
+        Ok(response)
     }
 
     pub async fn shutdown(&self) {
@@ -630,8 +642,23 @@ impl DesktopRuntime {
         self.recording_manager.delete(id).await
     }
 
-    pub async fn start_streaming(&self) -> DesktopStreamStatusResponse {
+    pub async fn start_streaming(&self) -> Result<DesktopStreamStatusResponse, SandboxError> {
-        self.streaming_manager.start().await
+        let state = self.inner.lock().await;
+        let display = state
+            .display
+            .as_deref()
+            .ok_or_else(|| SandboxError::Conflict {
+                message: "desktop runtime is not active".to_string(),
+            })?;
+        let resolution = state
+            .resolution
+            .clone()
+            .ok_or_else(|| SandboxError::Conflict {
+                message: "desktop runtime is not active".to_string(),
+            })?;
+        let display = display.to_string();
+        drop(state);
+        Ok(self.streaming_manager.start(&display, resolution).await)
     }
 
     pub async fn stop_streaming(&self) -> DesktopStreamStatusResponse {
@@ -639,7 +666,17 @@ impl DesktopRuntime {
     }
 
     pub async fn ensure_streaming_active(&self) -> Result<(), SandboxError> {
-        self.streaming_manager.ensure_active().await
+        if self.streaming_manager.is_active().await {
+            Ok(())
+        } else {
+            Err(SandboxError::Conflict {
+                message: "desktop streaming is not active".to_string(),
+            })
+        }
+    }
+
+    pub fn streaming_manager(&self) -> &DesktopStreamingManager {
+        &self.streaming_manager
     }
 
     async fn recording_context(&self) -> Result<DesktopRecordingContext, SandboxError> {
@@ -831,8 +868,14 @@ impl DesktopRuntime {
         name: &str,
     ) -> Result<(), DesktopProblem> {
         let process_id = match name {
-            "Xvfb" => state.xvfb.as_ref().map(|process| process.process_id.clone()),
+            "Xvfb" => state
-            "openbox" => state.openbox.as_ref().map(|process| process.process_id.clone()),
+                .xvfb
+                .as_ref()
+                .map(|process| process.process_id.clone()),
+            "openbox" => state
+                .openbox
+                .as_ref()
+                .map(|process| process.process_id.clone()),
             _ => None,
         };
 

server/packages/sandbox-agent/src/desktop_streaming.rs

@@ -2,9 +2,7 @@ use std::sync::Arc;
 
 use tokio::sync::Mutex;
 
-use sandbox_agent_error::SandboxError;
+use crate::desktop_types::{DesktopResolution, DesktopStreamStatusResponse};
-
-use crate::desktop_types::DesktopStreamStatusResponse;
 
 #[derive(Debug, Clone)]
 pub struct DesktopStreamingManager {
@@ -14,6 +12,8 @@ pub struct DesktopStreamingManager {
 #[derive(Debug, Default)]
 struct DesktopStreamingState {
     active: bool,
+    display: Option<String>,
+    resolution: Option<DesktopResolution>,
 }
 
 impl DesktopStreamingManager {
@@ -23,25 +23,46 @@ impl DesktopStreamingManager {
         }
     }
 
-    pub async fn start(&self) -> DesktopStreamStatusResponse {
+    /// Mark desktop streaming as active for the given display and resolution.
+    ///
+    /// The actual GStreamer pipeline is created per-WebSocket-session in the
+    /// signaling handler — this method just records that streaming is enabled.
+    pub async fn start(
+        &self,
+        display: &str,
+        resolution: DesktopResolution,
+    ) -> DesktopStreamStatusResponse {
         let mut state = self.inner.lock().await;
+
+        if state.active {
+            return DesktopStreamStatusResponse { active: true };
+        }
+
         state.active = true;
+        state.display = Some(display.to_string());
+        state.resolution = Some(resolution);
+
         DesktopStreamStatusResponse { active: true }
     }
 
+    /// Stop streaming and clear state.
     pub async fn stop(&self) -> DesktopStreamStatusResponse {
         let mut state = self.inner.lock().await;
         state.active = false;
+        state.display = None;
+        state.resolution = None;
         DesktopStreamStatusResponse { active: false }
     }
 
-    pub async fn ensure_active(&self) -> Result<(), SandboxError> {
+    pub async fn is_active(&self) -> bool {
-        if self.inner.lock().await.active {
+        self.inner.lock().await.active
-            Ok(())
+    }
-        } else {
+
-            Err(SandboxError::Conflict {
+    pub async fn resolution(&self) -> Option<DesktopResolution> {
-                message: "desktop streaming is not active".to_string(),
+        self.inner.lock().await.resolution.clone()
-            })
+    }
-        }
+
+    pub async fn display_name(&self) -> Option<String> {
+        self.inner.lock().await.display.clone()
     }
 }

server/packages/sandbox-agent/src/lib.rs

@@ -4,6 +4,7 @@ mod acp_proxy_runtime;
 pub mod cli;
 pub mod daemon;
 mod desktop_errors;
+mod desktop_gstreamer;
 mod desktop_install;
 mod desktop_recording;
 mod desktop_runtime;

server/packages/sandbox-agent/src/router.rs

@@ -41,9 +41,9 @@ use crate::desktop_errors::DesktopProblem;
 use crate::desktop_runtime::DesktopRuntime;
 use crate::desktop_types::*;
 use crate::process_runtime::{
-    decode_input_bytes, ProcessLogFilter, ProcessLogFilterStream, ProcessOwner as RuntimeProcessOwner,
+    decode_input_bytes, ProcessLogFilter, ProcessLogFilterStream,
-    ProcessRuntime, ProcessRuntimeConfig, ProcessSnapshot, ProcessStartSpec, ProcessStatus,
+    ProcessOwner as RuntimeProcessOwner, ProcessRuntime, ProcessRuntimeConfig, ProcessSnapshot,
-    ProcessStream, RunSpec,
+    ProcessStartSpec, ProcessStatus, ProcessStream, RunSpec,
 };
 use crate::ui;
 
@@ -235,7 +235,7 @@ pub fn build_router_with_state(shared: Arc<AppState>) -> (Router, Arc<AppState>)
         )
         .route("/desktop/stream/start", post(post_v1_desktop_stream_start))
         .route("/desktop/stream/stop", post(post_v1_desktop_stream_stop))
-        .route("/desktop/stream/ws", get(get_v1_desktop_stream_ws))
+        .route("/desktop/stream/signaling", get(get_v1_desktop_stream_ws))
         .route("/agents", get(get_v1_agents))
         .route("/agents/:agent", get(get_v1_agent))
         .route("/agents/:agent/install", post(post_v1_agent_install))
@@ -1135,9 +1135,11 @@ async fn get_v1_desktop_recording_download(
     Path(id): Path<String>,
 ) -> Result<Response, ApiError> {
     let path = state.desktop_runtime().recording_download_path(&id).await?;
-    let bytes = tokio::fs::read(&path).await.map_err(|err| SandboxError::StreamError {
+    let bytes = tokio::fs::read(&path)
-        message: format!("failed to read desktop recording {}: {err}", path.display()),
+        .await
-    })?;
+        .map_err(|err| SandboxError::StreamError {
+            message: format!("failed to read desktop recording {}: {err}", path.display()),
+        })?;
     Ok(([(header::CONTENT_TYPE, "video/mp4")], Bytes::from(bytes)).into_response())
 }
 
@@ -1179,7 +1181,7 @@ async fn delete_v1_desktop_recording(
 async fn post_v1_desktop_stream_start(
     State(state): State<Arc<AppState>>,
 ) -> Result<Json<DesktopStreamStatusResponse>, ApiError> {
-    Ok(Json(state.desktop_runtime().start_streaming().await))
+    Ok(Json(state.desktop_runtime().start_streaming().await?))
 }
 
 /// Stop desktop streaming.
@@ -1199,13 +1201,14 @@ async fn post_v1_desktop_stream_stop(
     Ok(Json(state.desktop_runtime().stop_streaming().await))
 }
 
-/// Open a desktop websocket streaming session.
+/// Open a desktop WebRTC signaling session.
 ///
-/// Upgrades the connection to a websocket that streams JPEG desktop frames and
+/// Upgrades the connection to a WebSocket used for WebRTC signaling between
-/// accepts mouse and keyboard control frames.
+/// the browser client and the desktop streaming process. Also accepts mouse
+/// and keyboard input frames as a fallback transport.
 #[utoipa::path(
     get,
-    path = "/v1/desktop/stream/ws",
+    path = "/v1/desktop/stream/signaling",
     tag = "v1",
     params(
         ("access_token" = Option<String>, Query, description = "Bearer token alternative for WS auth")
@@ -2449,46 +2452,6 @@ enum TerminalClientFrame {
     Close,
 }
 
-#[derive(Debug, Deserialize)]
-#[serde(tag = "type", rename_all = "camelCase")]
-enum DesktopStreamClientFrame {
-    MoveMouse {
-        x: i32,
-        y: i32,
-    },
-    MouseDown {
-        #[serde(default)]
-        x: Option<i32>,
-        #[serde(default)]
-        y: Option<i32>,
-        #[serde(default)]
-        button: Option<DesktopMouseButton>,
-    },
-    MouseUp {
-        #[serde(default)]
-        x: Option<i32>,
-        #[serde(default)]
-        y: Option<i32>,
-        #[serde(default)]
-        button: Option<DesktopMouseButton>,
-    },
-    Scroll {
-        x: i32,
-        y: i32,
-        #[serde(default)]
-        delta_x: Option<i32>,
-        #[serde(default)]
-        delta_y: Option<i32>,
-    },
-    KeyDown {
-        key: String,
-    },
-    KeyUp {
-        key: String,
-    },
-    Close,
-}
-
 async fn process_terminal_ws_session(
     mut socket: WebSocket,
     runtime: Arc<ProcessRuntime>,
@@ -2601,22 +2564,38 @@ async fn process_terminal_ws_session(
     }
 }
 
-async fn desktop_stream_ws_session(mut socket: WebSocket, desktop_runtime: Arc<DesktopRuntime>) {
+/// WebRTC signaling and input session.
-    let display_info = match desktop_runtime.display_info().await {
+///
-        Ok(info) => info,
+/// Handles WebRTC signaling (SDP offer/answer, ICE candidate exchange) and
-        Err(err) => {
+/// accepts mouse/keyboard input as a fallback transport when the WebRTC data
-            let _ = send_ws_error(&mut socket, &err.to_error_info().message).await;
+/// channel is not established. When compiled with the `desktop-gstreamer`
-            let _ = socket.close().await;
+/// feature, creates a GStreamer pipeline for real video streaming.
-            return;
+async fn desktop_stream_ws_session(mut ws: WebSocket, desktop_runtime: Arc<DesktopRuntime>) {
-        }
+    let streaming = desktop_runtime.streaming_manager();
-    };
 
+    // Get resolution for the ready message.
+    let resolution =
+        streaming
+            .resolution()
+            .await
+            .unwrap_or(crate::desktop_types::DesktopResolution {
+                width: 1440,
+                height: 900,
+                dpi: None,
+            });
+
+    let x_display = streaming
+        .display_name()
+        .await
+        .unwrap_or_else(|| ":99".to_string());
+
+    // Send stream metadata immediately.
     if send_ws_json(
-        &mut socket,
+        &mut ws,
         json!({
             "type": "ready",
-            "width": display_info.resolution.width,
+            "width": resolution.width,
-            "height": display_info.resolution.height,
+            "height": resolution.height,
         }),
     )
     .await
@@ -2625,109 +2604,270 @@ async fn desktop_stream_ws_session(mut socket: WebSocket, desktop_runtime: Arc<D
         return;
     }
 
-    let mut frame_tick = tokio::time::interval(Duration::from_millis(100));
+    // Try to create a GStreamer WebRTC pipeline for real video streaming.
+    #[cfg(feature = "desktop-gstreamer")]
+    {
+        use crate::desktop_gstreamer::pipeline::GStreamerPipeline;
+
+        match GStreamerPipeline::new(&x_display) {
+            Ok((pipeline, mut event_rx)) => {
+                tracing::info!(display = %x_display, "GStreamer WebRTC pipeline started");
+                // Run the session with the GStreamer pipeline active.
+                desktop_stream_ws_loop_gstreamer(
+                    &mut ws,
+                    &desktop_runtime,
+                    &pipeline,
+                    &mut event_rx,
+                )
+                .await;
+                // Pipeline is dropped here, stopping GStreamer.
+                let _ = ws.close().await;
+                return;
+            }
+            Err(e) => {
+                tracing::warn!(error = %e, "GStreamer pipeline creation failed");
+                let _ = send_ws_error(&mut ws, &e).await;
+            }
+        }
+    }
+
+    // Fallback: run without GStreamer (input-only, no video).
+    desktop_stream_ws_loop_simple(&mut ws, &desktop_runtime).await;
+    let _ = ws.close().await;
+}
+
+/// Inner WS message loop — input-only, no GStreamer pipeline.
+async fn desktop_stream_ws_loop_simple(ws: &mut WebSocket, desktop_runtime: &Arc<DesktopRuntime>) {
+    loop {
+        let ws_msg = ws.recv().await;
+        if !handle_ws_message_simple(ws_msg, ws, desktop_runtime).await {
+            break;
+        }
+    }
+}
+
+/// Inner WS message loop with GStreamer pipeline — polls both pipeline events
+/// and client WS messages.
+#[cfg(feature = "desktop-gstreamer")]
+async fn desktop_stream_ws_loop_gstreamer(
+    ws: &mut WebSocket,
+    desktop_runtime: &Arc<DesktopRuntime>,
+    pipeline: &crate::desktop_gstreamer::pipeline::GStreamerPipeline,
+    event_rx: &mut tokio::sync::mpsc::UnboundedReceiver<
+        crate::desktop_gstreamer::pipeline::PipelineEvent,
+    >,
+) {
+    use crate::desktop_gstreamer::pipeline::{PipelineEvent, SignalingCommand};
 
     loop {
         tokio::select! {
-            ws_in = socket.recv() => {
+            pipeline_event = event_rx.recv() => {
-                match ws_in {
+                match pipeline_event {
-                    Some(Ok(Message::Text(text))) => {
+                    Some(PipelineEvent::Offer(sdp)) => {
-                        match serde_json::from_str::<DesktopStreamClientFrame>(&text) {
+                        if send_ws_json(ws, json!({"type": "offer", "sdp": sdp})).await.is_err() {
-                            Ok(DesktopStreamClientFrame::MoveMouse { x, y }) => {
-                                if let Err(err) = desktop_runtime
-                                    .move_mouse(DesktopMouseMoveRequest { x, y })
-                                    .await
-                                {
-                                    let _ = send_ws_error(&mut socket, &err.to_error_info().message).await;
-                                }
-                            }
-                            Ok(DesktopStreamClientFrame::MouseDown { x, y, button }) => {
-                                if let Err(err) = desktop_runtime
-                                    .mouse_down(DesktopMouseDownRequest { x, y, button })
-                                    .await
-                                {
-                                    let _ = send_ws_error(&mut socket, &err.to_error_info().message).await;
-                                }
-                            }
-                            Ok(DesktopStreamClientFrame::MouseUp { x, y, button }) => {
-                                if let Err(err) = desktop_runtime
-                                    .mouse_up(DesktopMouseUpRequest { x, y, button })
-                                    .await
-                                {
-                                    let _ = send_ws_error(&mut socket, &err.to_error_info().message).await;
-                                }
-                            }
-                            Ok(DesktopStreamClientFrame::Scroll { x, y, delta_x, delta_y }) => {
-                                if let Err(err) = desktop_runtime
-                                    .scroll_mouse(DesktopMouseScrollRequest {
-                                        x,
-                                        y,
-                                        delta_x,
-                                        delta_y,
-                                    })
-                                    .await
-                                {
-                                    let _ = send_ws_error(&mut socket, &err.to_error_info().message).await;
-                                }
-                            }
-                            Ok(DesktopStreamClientFrame::KeyDown { key }) => {
-                                if let Err(err) = desktop_runtime
-                                    .key_down(DesktopKeyboardDownRequest { key })
-                                    .await
-                                {
-                                    let _ = send_ws_error(&mut socket, &err.to_error_info().message).await;
-                                }
-                            }
-                            Ok(DesktopStreamClientFrame::KeyUp { key }) => {
-                                if let Err(err) = desktop_runtime
-                                    .key_up(DesktopKeyboardUpRequest { key })
-                                    .await
-                                {
-                                    let _ = send_ws_error(&mut socket, &err.to_error_info().message).await;
-                                }
-                            }
-                            Ok(DesktopStreamClientFrame::Close) => {
-                                let _ = socket.close().await;
-                                break;
-                            }
-                            Err(err) => {
-                                let _ = send_ws_error(&mut socket, &format!("invalid desktop stream frame: {err}")).await;
-                            }
-                        }
-                    }
-                    Some(Ok(Message::Ping(payload))) => {
-                        let _ = socket.send(Message::Pong(payload)).await;
-                    }
-                    Some(Ok(Message::Close(_))) | None => break,
-                    Some(Ok(Message::Binary(_))) | Some(Ok(Message::Pong(_))) => {}
-                    Some(Err(_)) => break,
-                }
-            }
-            _ = frame_tick.tick() => {
-                let frame = desktop_runtime
-                    .screenshot(DesktopScreenshotQuery {
-                        format: Some(DesktopScreenshotFormat::Jpeg),
-                        quality: Some(60),
-                        scale: Some(1.0),
-                    })
-                    .await;
-                match frame {
-                    Ok(frame) => {
-                        if socket.send(Message::Binary(frame.bytes.into())).await.is_err() {
                             break;
                         }
                     }
-                    Err(err) => {
+                    Some(PipelineEvent::IceCandidate { candidate, sdp_m_line_index }) => {
-                        let _ = send_ws_error(&mut socket, &err.to_error_info().message).await;
+                        if send_ws_json(ws, json!({
-                        let _ = socket.close().await;
+                            "type": "candidate",
-                        break;
+                            "candidate": candidate,
+                            "sdpMLineIndex": sdp_m_line_index,
+                        })).await.is_err() {
+                            break;
+                        }
                     }
+                    None => break,
+                }
+            }
+            ws_msg = ws.recv() => {
+                match ws_msg {
+                    Some(Ok(Message::Text(text))) => {
+                        let parsed: Value = match serde_json::from_str(&text) {
+                            Ok(v) => v,
+                            Err(_) => continue,
+                        };
+                        match parsed.get("type").and_then(|v| v.as_str()) {
+                            Some("answer") => {
+                                if let Some(sdp) = parsed.get("sdp").and_then(|v| v.as_str()) {
+                                    pipeline.send_command(SignalingCommand::Answer(sdp.to_string()));
+                                }
+                            }
+                            Some("candidate") => {
+                                if let Some(candidate) = parsed.get("candidate").and_then(|v| v.as_str()) {
+                                    let sdp_m_line_index = parsed
+                                        .get("sdpMLineIndex")
+                                        .and_then(|v| v.as_u64())
+                                        .unwrap_or(0) as u32;
+                                    pipeline.send_command(SignalingCommand::IceCandidate {
+                                        candidate: candidate.to_string(),
+                                        sdp_m_line_index,
+                                    });
+                                }
+                            }
+                            // Input messages (fallback transport)
+                            Some("moveMouse") => {
+                                if let (Some(x), Some(y)) = (
+                                    parsed.get("x").and_then(|v| v.as_i64()),
+                                    parsed.get("y").and_then(|v| v.as_i64()),
+                                ) {
+                                    let _ = desktop_runtime
+                                        .move_mouse(DesktopMouseMoveRequest { x: x as i32, y: y as i32 })
+                                        .await;
+                                }
+                            }
+                            Some("mouseDown") => {
+                                let button = parsed.get("button").and_then(|v| serde_json::from_value(v.clone()).ok());
+                                let x = parsed.get("x").and_then(|v| v.as_i64()).map(|v| v as i32);
+                                let y = parsed.get("y").and_then(|v| v.as_i64()).map(|v| v as i32);
+                                let _ = desktop_runtime.mouse_down(DesktopMouseDownRequest { x, y, button }).await;
+                            }
+                            Some("mouseUp") => {
+                                let button = parsed.get("button").and_then(|v| serde_json::from_value(v.clone()).ok());
+                                let x = parsed.get("x").and_then(|v| v.as_i64()).map(|v| v as i32);
+                                let y = parsed.get("y").and_then(|v| v.as_i64()).map(|v| v as i32);
+                                let _ = desktop_runtime.mouse_up(DesktopMouseUpRequest { x, y, button }).await;
+                            }
+                            Some("scroll") => {
+                                if let (Some(x), Some(y)) = (
+                                    parsed.get("x").and_then(|v| v.as_i64()),
+                                    parsed.get("y").and_then(|v| v.as_i64()),
+                                ) {
+                                    let dx = parsed.get("deltaX").and_then(|v| v.as_i64()).map(|v| v as i32);
+                                    let dy = parsed.get("deltaY").and_then(|v| v.as_i64()).map(|v| v as i32);
+                                    let _ = desktop_runtime.scroll_mouse(DesktopMouseScrollRequest { x: x as i32, y: y as i32, delta_x: dx, delta_y: dy }).await;
+                                }
+                            }
+                            Some("keyDown") => {
+                                if let Some(key) = parsed.get("key").and_then(|v| v.as_str()) {
+                                    let _ = desktop_runtime.key_down(DesktopKeyboardDownRequest { key: key.to_string() }).await;
+                                }
+                            }
+                            Some("keyUp") => {
+                                if let Some(key) = parsed.get("key").and_then(|v| v.as_str()) {
+                                    let _ = desktop_runtime.key_up(DesktopKeyboardUpRequest { key: key.to_string() }).await;
+                                }
+                            }
+                            _ => {}
+                        }
+                    }
+                    Some(Ok(Message::Ping(payload))) => {
+                        let _ = ws.send(Message::Pong(payload)).await;
+                    }
+                    Some(Ok(Message::Close(_))) | None | Some(Err(_)) => break,
+                    _ => {}
                 }
             }
         }
     }
 }
 
+/// Process a single WebSocket message (no pipeline). Returns false to close.
+async fn handle_ws_message_simple(
+    msg: Option<Result<Message, axum::Error>>,
+    ws: &mut WebSocket,
+    desktop_runtime: &Arc<DesktopRuntime>,
+) -> bool {
+    match msg {
+        Some(Ok(Message::Text(text))) => {
+            let parsed: Value = match serde_json::from_str(&text) {
+                Ok(v) => v,
+                Err(_) => return true,
+            };
+
+            match parsed.get("type").and_then(|v| v.as_str()) {
+                // --- Input messages (fallback transport) ---
+                Some("moveMouse") => {
+                    if let (Some(x), Some(y)) = (
+                        parsed.get("x").and_then(|v| v.as_i64()),
+                        parsed.get("y").and_then(|v| v.as_i64()),
+                    ) {
+                        let _ = desktop_runtime
+                            .move_mouse(DesktopMouseMoveRequest {
+                                x: x as i32,
+                                y: y as i32,
+                            })
+                            .await;
+                    }
+                }
+                Some("mouseDown") => {
+                    let button = parsed
+                        .get("button")
+                        .and_then(|v| serde_json::from_value(v.clone()).ok());
+                    let x = parsed.get("x").and_then(|v| v.as_i64()).map(|v| v as i32);
+                    let y = parsed.get("y").and_then(|v| v.as_i64()).map(|v| v as i32);
+                    let _ = desktop_runtime
+                        .mouse_down(DesktopMouseDownRequest { x, y, button })
+                        .await;
+                }
+                Some("mouseUp") => {
+                    let button = parsed
+                        .get("button")
+                        .and_then(|v| serde_json::from_value(v.clone()).ok());
+                    let x = parsed.get("x").and_then(|v| v.as_i64()).map(|v| v as i32);
+                    let y = parsed.get("y").and_then(|v| v.as_i64()).map(|v| v as i32);
+                    let _ = desktop_runtime
+                        .mouse_up(DesktopMouseUpRequest { x, y, button })
+                        .await;
+                }
+                Some("scroll") => {
+                    if let (Some(x), Some(y)) = (
+                        parsed.get("x").and_then(|v| v.as_i64()),
+                        parsed.get("y").and_then(|v| v.as_i64()),
+                    ) {
+                        let delta_x = parsed
+                            .get("deltaX")
+                            .and_then(|v| v.as_i64())
+                            .map(|v| v as i32);
+                        let delta_y = parsed
+                            .get("deltaY")
+                            .and_then(|v| v.as_i64())
+                            .map(|v| v as i32);
+                        let _ = desktop_runtime
+                            .scroll_mouse(DesktopMouseScrollRequest {
+                                x: x as i32,
+                                y: y as i32,
+                                delta_x,
+                                delta_y,
+                            })
+                            .await;
+                    }
+                }
+                Some("keyDown") => {
+                    if let Some(key) = parsed.get("key").and_then(|v| v.as_str()) {
+                        let _ = desktop_runtime
+                            .key_down(DesktopKeyboardDownRequest {
+                                key: key.to_string(),
+                            })
+                            .await;
+                    }
+                }
+                Some("keyUp") => {
+                    if let Some(key) = parsed.get("key").and_then(|v| v.as_str()) {
+                        let _ = desktop_runtime
+                            .key_up(DesktopKeyboardUpRequest {
+                                key: key.to_string(),
+                            })
+                            .await;
+                    }
+                }
+
+                // --- WebRTC signaling messages (accepted without error) ---
+                Some("answer") | Some("candidate") | Some("offer") => {}
+
+                _ => {}
+            }
+            true
+        }
+        Some(Ok(Message::Ping(payload))) => {
+            let _ = ws.send(Message::Pong(payload)).await;
+            true
+        }
+        Some(Ok(Message::Close(_))) | None | Some(Err(_)) => false,
+        _ => true,
+    }
+}
+
 async fn send_ws_json(socket: &mut WebSocket, payload: Value) -> Result<(), ()> {
     socket
         .send(Message::Text(

server/packages/sandbox-agent/tests/v1_api/desktop.rs

@@ -432,7 +432,7 @@ async fn v1_desktop_lifecycle_and_actions_work_with_real_runtime() {
     assert_eq!(status, StatusCode::OK);
     assert_eq!(parse_json(&body)["active"], true);
 
-    let (mut ws, _) = connect_async(test_app.app.ws_url("/v1/desktop/stream/ws"))
+    let (mut ws, _) = connect_async(test_app.app.ws_url("/v1/desktop/stream/signaling"))
         .await
         .expect("connect desktop stream websocket");
 
@@ -447,12 +447,9 @@ async fn v1_desktop_lifecycle_and_actions_work_with_real_runtime() {
         other => panic!("expected text ready frame, got {other:?}"),
     }
 
-    let frame = recv_ws_message(&mut ws).await;
+    // The signaling WebSocket now accepts input frames as fallback transport
-    match frame {
+    // (when the WebRTC data channel is not established). Send a mouse move to
-        Message::Binary(bytes) => assert!(bytes.starts_with(&[0xff, 0xd8, 0xff])),
+    // verify input dispatch still works over the signaling channel.
-        other => panic!("expected binary jpeg frame, got {other:?}"),
-    }
-
     ws.send(Message::Text(
         json!({
             "type": "moveMouse",
@@ -464,6 +461,20 @@ async fn v1_desktop_lifecycle_and_actions_work_with_real_runtime() {
     ))
     .await
     .expect("send desktop stream mouse move");
+
+    // Send a WebRTC signaling message (offer) to verify the signaling path
+    // accepts it without error.
+    ws.send(Message::Text(
+        json!({
+            "type": "offer",
+            "sdp": "v=0\r\n"
+        })
+        .to_string()
+        .into(),
+    ))
+    .await
+    .expect("send desktop stream offer");
+
     let _ = ws.close(None).await;
 
     let (status, _, body) = send_request(