fix: address PR #264 review issues

- Fix connectDesktopStream silently dropping RTCPeerConnection and rtcConfig options (client.ts)
- Fix DesktopViewer useEffect dependency causing reconnect loop (store callbacks in refs)
- Fix TOCTOU race condition in DesktopRecordingManager::start() (merge lock scope)
- Fix incomplete cursor bounds check in composite_cursor_region (add right/bottom checks)
- Add DesktopViewer to react-components.mdx documentation
- Remove hardcoded visual styles from DesktopViewer (make unstyled by default per sdks/CLAUDE.md)
- Export DesktopViewerClassNames type for consumer styling

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
Nathan Flurry 2026-03-17 02:49:03 -07:00
parent f25a92aca8
commit bdd7526de5
6 changed files with 101 additions and 79 deletions

View file

@ -12,6 +12,7 @@ Current exports:
- `ProcessTerminal` for attaching to a running tty process
- `AgentTranscript` for rendering session/message timelines without bundling any styles
- `ChatComposer` for a reusable prompt input/send surface
- `DesktopViewer` for rendering a live desktop stream with mouse and keyboard input
- `useTranscriptVirtualizer` for wiring large transcript lists to a scroll container
## Install
@ -243,3 +244,47 @@ Useful `ChatComposer` props:
- `allowEmptySubmit` when the submit action is valid without draft text, such as a stop button
Use `transcriptProps` and `composerProps` when you want the shared composition but still need custom rendering or behavior. Use `transcriptClassNames` and `composerClassNames` when you want styling hooks for each subcomponent.
## Desktop viewer
`DesktopViewer` connects to a live desktop stream via WebRTC and renders the video feed with interactive mouse and keyboard input forwarding.
```tsx DesktopPane.tsx
"use client";
import { useEffect, useState } from "react";
import { SandboxAgent } from "sandbox-agent";
import { DesktopViewer } from "@sandbox-agent/react";
export default function DesktopPane() {
const [client, setClient] = useState<SandboxAgent | null>(null);
useEffect(() => {
let sdk: SandboxAgent | null = null;
const start = async () => {
sdk = await SandboxAgent.connect({ baseUrl: "http://127.0.0.1:2468" });
await sdk.startDesktop();
await sdk.startDesktopStream();
setClient(sdk);
};
void start();
return () => { void sdk?.dispose(); };
}, []);
if (!client) return <div>Starting desktop...</div>;
return <DesktopViewer client={client} height={600} />;
}
```
Props:
- `client`: a `SandboxAgent` client (or any object with `connectDesktopStream`)
- `height`, `style`, `imageStyle`: optional layout overrides
- `showStatusBar`: toggle the connection status bar (default `true`)
- `onConnect`, `onDisconnect`, `onError`: optional lifecycle callbacks
- `className` and `classNames`: external styling hooks
The component is unstyled by default. Use `classNames` slots (`root`, `statusBar`, `statusText`, `statusResolution`, `viewport`, `video`) and `data-slot`/`data-state` attributes for styling from outside the package.
See [Computer Use](/computer-use) for the lower-level desktop APIs.

View file

@ -8,9 +8,19 @@ type ConnectionState = "connecting" | "ready" | "closed" | "error";
export type DesktopViewerClient = Pick<SandboxAgent, "connectDesktopStream">;
export interface DesktopViewerClassNames {
root?: string;
statusBar?: string;
statusText?: string;
statusResolution?: string;
viewport?: string;
video?: string;
}
export interface DesktopViewerProps {
client: DesktopViewerClient;
className?: string;
classNames?: Partial<DesktopViewerClassNames>;
style?: CSSProperties;
imageStyle?: CSSProperties;
height?: number | string;
@ -20,66 +30,17 @@ export interface DesktopViewerProps {
onError?: (error: DesktopStreamErrorStatus | Error) => void;
}
const shellStyle: CSSProperties = {
display: "flex",
flexDirection: "column",
overflow: "hidden",
border: "1px solid rgba(15, 23, 42, 0.14)",
borderRadius: 14,
background: "linear-gradient(180deg, rgba(248, 250, 252, 0.96) 0%, rgba(226, 232, 240, 0.92) 100%)",
boxShadow: "0 20px 40px rgba(15, 23, 42, 0.08)",
};
const statusBarStyle: CSSProperties = {
display: "flex",
alignItems: "center",
justifyContent: "space-between",
gap: 12,
padding: "10px 14px",
borderBottom: "1px solid rgba(15, 23, 42, 0.08)",
background: "rgba(255, 255, 255, 0.78)",
color: "#0f172a",
fontSize: 12,
lineHeight: 1.4,
};
const viewportStyle: CSSProperties = {
position: "relative",
display: "flex",
alignItems: "center",
justifyContent: "center",
overflow: "hidden",
background: "radial-gradient(circle at top, rgba(14, 165, 233, 0.18), transparent 45%), linear-gradient(180deg, #0f172a 0%, #111827 100%)",
};
const videoBaseStyle: CSSProperties = {
display: "block",
width: "100%",
height: "100%",
objectFit: "contain",
userSelect: "none",
};
const hintStyle: CSSProperties = {
opacity: 0.66,
};
const getStatusColor = (state: ConnectionState): string => {
switch (state) {
case "ready":
return "#15803d";
case "error":
return "#b91c1c";
case "closed":
return "#b45309";
default:
return "#475569";
}
const layoutStyles = {
shell: { display: "flex", flexDirection: "column", overflow: "hidden" } as CSSProperties,
statusBar: { display: "flex", alignItems: "center", justifyContent: "space-between", gap: 12 } as CSSProperties,
viewport: { position: "relative", display: "flex", alignItems: "center", justifyContent: "center", overflow: "hidden" } as CSSProperties,
video: { display: "block", width: "100%", height: "100%", objectFit: "contain", userSelect: "none" } as CSSProperties,
};
export const DesktopViewer = ({
client,
className,
classNames,
style,
imageStyle,
height = 480,
@ -91,11 +52,18 @@ export const DesktopViewer = ({
const wrapperRef = useRef<HTMLDivElement | null>(null);
const videoRef = useRef<HTMLVideoElement | null>(null);
const sessionRef = useRef<ReturnType<DesktopViewerClient["connectDesktopStream"]> | null>(null);
const onConnectRef = useRef(onConnect);
const onDisconnectRef = useRef(onDisconnect);
const onErrorRef = useRef(onError);
const [connectionState, setConnectionState] = useState<ConnectionState>("connecting");
const [statusMessage, setStatusMessage] = useState("Starting desktop stream...");
const [hasVideo, setHasVideo] = useState(false);
const [resolution, setResolution] = useState<{ width: number; height: number } | null>(null);
onConnectRef.current = onConnect;
onDisconnectRef.current = onDisconnect;
onErrorRef.current = onError;
useEffect(() => {
let cancelled = false;
@ -112,7 +80,7 @@ export const DesktopViewer = ({
setConnectionState("ready");
setStatusMessage("Desktop stream connected.");
setResolution({ width: status.width, height: status.height });
onConnect?.(status);
onConnectRef.current?.(status);
});
session.onTrack((stream) => {
if (cancelled) return;
@ -127,13 +95,13 @@ export const DesktopViewer = ({
if (cancelled) return;
setConnectionState("error");
setStatusMessage(error instanceof Error ? error.message : error.message);
onError?.(error);
onErrorRef.current?.(error);
});
session.onDisconnect(() => {
if (cancelled) return;
setConnectionState((current) => (current === "error" ? current : "closed"));
setStatusMessage((current) => (current === "Desktop stream connected." ? "Desktop stream disconnected." : current));
onDisconnect?.();
onDisconnectRef.current?.();
});
return () => {
@ -146,7 +114,7 @@ export const DesktopViewer = ({
}
setHasVideo(false);
};
}, [client, onConnect, onDisconnect, onError]);
}, [client]);
const scalePoint = (clientX: number, clientY: number) => {
const video = videoRef.current;
@ -204,18 +172,24 @@ export const DesktopViewer = ({
};
return (
<div className={className} style={{ ...shellStyle, ...style }}>
<div className={classNames?.root ?? className} data-slot="root" data-state={connectionState} style={{ ...layoutStyles.shell, ...style }}>
{showStatusBar ? (
<div style={statusBarStyle}>
<span style={{ color: getStatusColor(connectionState) }}>{statusMessage}</span>
<span style={hintStyle}>{resolution ? `${resolution.width}×${resolution.height}` : "Awaiting stream"}</span>
<div className={classNames?.statusBar} data-slot="status-bar" style={layoutStyles.statusBar}>
<span className={classNames?.statusText} data-slot="status-text" data-state={connectionState}>
{statusMessage}
</span>
<span className={classNames?.statusResolution} data-slot="status-resolution">
{resolution ? `${resolution.width}×${resolution.height}` : "Awaiting stream"}
</span>
</div>
) : null}
<div
ref={wrapperRef}
className={classNames?.viewport}
data-slot="viewport"
role="button"
tabIndex={0}
style={{ ...viewportStyle, height }}
style={{ ...layoutStyles.viewport, height }}
onMouseMove={(event) => {
const point = scalePoint(event.clientX, event.clientY);
if (!point) {
@ -259,12 +233,14 @@ export const DesktopViewer = ({
>
<video
ref={videoRef}
className={classNames?.video}
data-slot="video"
autoPlay
playsInline
muted
tabIndex={-1}
draggable={false}
style={{ ...videoBaseStyle, ...imageStyle, display: hasVideo ? "block" : "none", pointerEvents: "none" }}
style={{ ...layoutStyles.video, ...imageStyle, display: hasVideo ? "block" : "none", pointerEvents: "none" }}
/>
</div>
</div>

View file

@ -25,6 +25,7 @@ export type {
} from "./ChatComposer.tsx";
export type {
DesktopViewerClassNames,
DesktopViewerClient,
DesktopViewerProps,
} from "./DesktopViewer.tsx";

View file

@ -2005,7 +2005,7 @@ export class SandboxAgent {
}
connectDesktopStream(options: DesktopStreamSessionOptions = {}): DesktopStreamSession {
return new DesktopStreamSession(this.connectDesktopStreamWebSocket(options));
return new DesktopStreamSession(this.connectDesktopStreamWebSocket(options), options);
}
private async getLiveConnection(agent: string): Promise<LiveAcpConnection> {

View file

@ -64,7 +64,6 @@ impl DesktopRecordingManager {
self.ensure_recordings_dir()?;
{
let mut state = self.inner.lock().await;
self.refresh_locked(&mut state).await?;
if state.current_id.is_some() {
@ -72,9 +71,6 @@ impl DesktopRecordingManager {
message: "a desktop recording is already active".to_string(),
});
}
}
let mut state = self.inner.lock().await;
let id_num = state.next_id + 1;
state.next_id = id_num;
let id = format!("rec_{id_num}");

View file

@ -2036,14 +2036,18 @@ impl DesktopRuntime {
options: &DesktopScreenshotOptions,
region_x: i32,
region_y: i32,
_region_width: u32,
_region_height: u32,
region_width: u32,
region_height: u32,
) -> Result<Vec<u8>, DesktopProblem> {
let pos = self.mouse_position_locked(state, ready).await?;
// Adjust cursor position relative to the region
let cursor_x = pos.x - region_x;
let cursor_y = pos.y - region_y;
if cursor_x < 0 || cursor_y < 0 {
if cursor_x < 0
|| cursor_y < 0
|| cursor_x >= region_width as i32
|| cursor_y >= region_height as i32
{
// Cursor is outside the region, return screenshot as-is
return Ok(screenshot_bytes);
}