diff --git a/docs/react-components.mdx b/docs/react-components.mdx index 93183b2..68dd160 100644 --- a/docs/react-components.mdx +++ b/docs/react-components.mdx @@ -12,6 +12,7 @@ Current exports: - `ProcessTerminal` for attaching to a running tty process - `AgentTranscript` for rendering session/message timelines without bundling any styles - `ChatComposer` for a reusable prompt input/send surface +- `DesktopViewer` for rendering a live desktop stream with mouse and keyboard input - `useTranscriptVirtualizer` for wiring large transcript lists to a scroll container ## Install @@ -243,3 +244,47 @@ Useful `ChatComposer` props: - `allowEmptySubmit` when the submit action is valid without draft text, such as a stop button Use `transcriptProps` and `composerProps` when you want the shared composition but still need custom rendering or behavior. Use `transcriptClassNames` and `composerClassNames` when you want styling hooks for each subcomponent. + +## Desktop viewer + +`DesktopViewer` connects to a live desktop stream via WebRTC and renders the video feed with interactive mouse and keyboard input forwarding. + +```tsx DesktopPane.tsx +"use client"; + +import { useEffect, useState } from "react"; +import { SandboxAgent } from "sandbox-agent"; +import { DesktopViewer } from "@sandbox-agent/react"; + +export default function DesktopPane() { + const [client, setClient] = useState(null); + + useEffect(() => { + let sdk: SandboxAgent | null = null; + const start = async () => { + sdk = await SandboxAgent.connect({ baseUrl: "http://127.0.0.1:2468" }); + await sdk.startDesktop(); + await sdk.startDesktopStream(); + setClient(sdk); + }; + void start(); + return () => { void sdk?.dispose(); }; + }, []); + + if (!client) return
Starting desktop...
; + + return ; +} +``` + +Props: + +- `client`: a `SandboxAgent` client (or any object with `connectDesktopStream`) +- `height`, `style`, `imageStyle`: optional layout overrides +- `showStatusBar`: toggle the connection status bar (default `true`) +- `onConnect`, `onDisconnect`, `onError`: optional lifecycle callbacks +- `className` and `classNames`: external styling hooks + +The component is unstyled by default. Use `classNames` slots (`root`, `statusBar`, `statusText`, `statusResolution`, `viewport`, `video`) and `data-slot`/`data-state` attributes for styling from outside the package. + +See [Computer Use](/computer-use) for the lower-level desktop APIs. diff --git a/sdks/react/src/DesktopViewer.tsx b/sdks/react/src/DesktopViewer.tsx index 55e0794..669c0a6 100644 --- a/sdks/react/src/DesktopViewer.tsx +++ b/sdks/react/src/DesktopViewer.tsx @@ -8,9 +8,19 @@ type ConnectionState = "connecting" | "ready" | "closed" | "error"; export type DesktopViewerClient = Pick; +export interface DesktopViewerClassNames { + root?: string; + statusBar?: string; + statusText?: string; + statusResolution?: string; + viewport?: string; + video?: string; +} + export interface DesktopViewerProps { client: DesktopViewerClient; className?: string; + classNames?: Partial; style?: CSSProperties; imageStyle?: CSSProperties; height?: number | string; @@ -20,66 +30,17 @@ export interface DesktopViewerProps { onError?: (error: DesktopStreamErrorStatus | Error) => void; } -const shellStyle: CSSProperties = { - display: "flex", - flexDirection: "column", - overflow: "hidden", - border: "1px solid rgba(15, 23, 42, 0.14)", - borderRadius: 14, - background: "linear-gradient(180deg, rgba(248, 250, 252, 0.96) 0%, rgba(226, 232, 240, 0.92) 100%)", - boxShadow: "0 20px 40px rgba(15, 23, 42, 0.08)", -}; - -const statusBarStyle: CSSProperties = { - display: "flex", - alignItems: "center", - justifyContent: "space-between", - gap: 12, - padding: "10px 14px", - borderBottom: "1px solid rgba(15, 23, 42, 0.08)", - background: "rgba(255, 255, 255, 0.78)", - color: "#0f172a", - fontSize: 12, - lineHeight: 1.4, -}; - -const viewportStyle: CSSProperties = { - position: "relative", - display: "flex", - alignItems: "center", - justifyContent: "center", - overflow: "hidden", - background: "radial-gradient(circle at top, rgba(14, 165, 233, 0.18), transparent 45%), linear-gradient(180deg, #0f172a 0%, #111827 100%)", -}; - -const videoBaseStyle: CSSProperties = { - display: "block", - width: "100%", - height: "100%", - objectFit: "contain", - userSelect: "none", -}; - -const hintStyle: CSSProperties = { - opacity: 0.66, -}; - -const getStatusColor = (state: ConnectionState): string => { - switch (state) { - case "ready": - return "#15803d"; - case "error": - return "#b91c1c"; - case "closed": - return "#b45309"; - default: - return "#475569"; - } +const layoutStyles = { + shell: { display: "flex", flexDirection: "column", overflow: "hidden" } as CSSProperties, + statusBar: { display: "flex", alignItems: "center", justifyContent: "space-between", gap: 12 } as CSSProperties, + viewport: { position: "relative", display: "flex", alignItems: "center", justifyContent: "center", overflow: "hidden" } as CSSProperties, + video: { display: "block", width: "100%", height: "100%", objectFit: "contain", userSelect: "none" } as CSSProperties, }; export const DesktopViewer = ({ client, className, + classNames, style, imageStyle, height = 480, @@ -91,11 +52,18 @@ export const DesktopViewer = ({ const wrapperRef = useRef(null); const videoRef = useRef(null); const sessionRef = useRef | null>(null); + const onConnectRef = useRef(onConnect); + const onDisconnectRef = useRef(onDisconnect); + const onErrorRef = useRef(onError); const [connectionState, setConnectionState] = useState("connecting"); const [statusMessage, setStatusMessage] = useState("Starting desktop stream..."); const [hasVideo, setHasVideo] = useState(false); const [resolution, setResolution] = useState<{ width: number; height: number } | null>(null); + onConnectRef.current = onConnect; + onDisconnectRef.current = onDisconnect; + onErrorRef.current = onError; + useEffect(() => { let cancelled = false; @@ -112,7 +80,7 @@ export const DesktopViewer = ({ setConnectionState("ready"); setStatusMessage("Desktop stream connected."); setResolution({ width: status.width, height: status.height }); - onConnect?.(status); + onConnectRef.current?.(status); }); session.onTrack((stream) => { if (cancelled) return; @@ -127,13 +95,13 @@ export const DesktopViewer = ({ if (cancelled) return; setConnectionState("error"); setStatusMessage(error instanceof Error ? error.message : error.message); - onError?.(error); + onErrorRef.current?.(error); }); session.onDisconnect(() => { if (cancelled) return; setConnectionState((current) => (current === "error" ? current : "closed")); setStatusMessage((current) => (current === "Desktop stream connected." ? "Desktop stream disconnected." : current)); - onDisconnect?.(); + onDisconnectRef.current?.(); }); return () => { @@ -146,7 +114,7 @@ export const DesktopViewer = ({ } setHasVideo(false); }; - }, [client, onConnect, onDisconnect, onError]); + }, [client]); const scalePoint = (clientX: number, clientY: number) => { const video = videoRef.current; @@ -204,18 +172,24 @@ export const DesktopViewer = ({ }; return ( -
+
{showStatusBar ? ( -
- {statusMessage} - {resolution ? `${resolution.width}×${resolution.height}` : "Awaiting stream"} +
+ + {statusMessage} + + + {resolution ? `${resolution.width}×${resolution.height}` : "Awaiting stream"} +
) : null}
{ const point = scalePoint(event.clientX, event.clientY); if (!point) { @@ -259,12 +233,14 @@ export const DesktopViewer = ({ >
diff --git a/sdks/react/src/index.ts b/sdks/react/src/index.ts index 1d8d1e1..f586ab3 100644 --- a/sdks/react/src/index.ts +++ b/sdks/react/src/index.ts @@ -25,6 +25,7 @@ export type { } from "./ChatComposer.tsx"; export type { + DesktopViewerClassNames, DesktopViewerClient, DesktopViewerProps, } from "./DesktopViewer.tsx"; diff --git a/sdks/typescript/src/client.ts b/sdks/typescript/src/client.ts index df66400..8a36455 100644 --- a/sdks/typescript/src/client.ts +++ b/sdks/typescript/src/client.ts @@ -2005,7 +2005,7 @@ export class SandboxAgent { } connectDesktopStream(options: DesktopStreamSessionOptions = {}): DesktopStreamSession { - return new DesktopStreamSession(this.connectDesktopStreamWebSocket(options)); + return new DesktopStreamSession(this.connectDesktopStreamWebSocket(options), options); } private async getLiveConnection(agent: string): Promise { diff --git a/server/packages/sandbox-agent/src/desktop_recording.rs b/server/packages/sandbox-agent/src/desktop_recording.rs index 39b174a..808e921 100644 --- a/server/packages/sandbox-agent/src/desktop_recording.rs +++ b/server/packages/sandbox-agent/src/desktop_recording.rs @@ -64,17 +64,13 @@ impl DesktopRecordingManager { self.ensure_recordings_dir()?; - { - let mut state = self.inner.lock().await; - self.refresh_locked(&mut state).await?; - if state.current_id.is_some() { - return Err(SandboxError::Conflict { - message: "a desktop recording is already active".to_string(), - }); - } - } - let mut state = self.inner.lock().await; + self.refresh_locked(&mut state).await?; + if state.current_id.is_some() { + return Err(SandboxError::Conflict { + message: "a desktop recording is already active".to_string(), + }); + } let id_num = state.next_id + 1; state.next_id = id_num; let id = format!("rec_{id_num}"); diff --git a/server/packages/sandbox-agent/src/desktop_runtime.rs b/server/packages/sandbox-agent/src/desktop_runtime.rs index 29a84dc..60c48af 100644 --- a/server/packages/sandbox-agent/src/desktop_runtime.rs +++ b/server/packages/sandbox-agent/src/desktop_runtime.rs @@ -2036,14 +2036,18 @@ impl DesktopRuntime { options: &DesktopScreenshotOptions, region_x: i32, region_y: i32, - _region_width: u32, - _region_height: u32, + region_width: u32, + region_height: u32, ) -> Result, DesktopProblem> { let pos = self.mouse_position_locked(state, ready).await?; // Adjust cursor position relative to the region let cursor_x = pos.x - region_x; let cursor_y = pos.y - region_y; - if cursor_x < 0 || cursor_y < 0 { + if cursor_x < 0 + || cursor_y < 0 + || cursor_x >= region_width as i32 + || cursor_y >= region_height as i32 + { // Cursor is outside the region, return screenshot as-is return Ok(screenshot_bytes); }