mirror of
https://github.com/harivansh-afk/sandbox-agent.git
synced 2026-04-15 06:04:43 +00:00
fix: address PR #264 review issues
- Fix connectDesktopStream silently dropping RTCPeerConnection and rtcConfig options (client.ts) - Fix DesktopViewer useEffect dependency causing reconnect loop (store callbacks in refs) - Fix TOCTOU race condition in DesktopRecordingManager::start() (merge lock scope) - Fix incomplete cursor bounds check in composite_cursor_region (add right/bottom checks) - Add DesktopViewer to react-components.mdx documentation - Remove hardcoded visual styles from DesktopViewer (make unstyled by default per sdks/CLAUDE.md) - Export DesktopViewerClassNames type for consumer styling Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
f25a92aca8
commit
bdd7526de5
6 changed files with 101 additions and 79 deletions
|
|
@ -12,6 +12,7 @@ Current exports:
|
||||||
- `ProcessTerminal` for attaching to a running tty process
|
- `ProcessTerminal` for attaching to a running tty process
|
||||||
- `AgentTranscript` for rendering session/message timelines without bundling any styles
|
- `AgentTranscript` for rendering session/message timelines without bundling any styles
|
||||||
- `ChatComposer` for a reusable prompt input/send surface
|
- `ChatComposer` for a reusable prompt input/send surface
|
||||||
|
- `DesktopViewer` for rendering a live desktop stream with mouse and keyboard input
|
||||||
- `useTranscriptVirtualizer` for wiring large transcript lists to a scroll container
|
- `useTranscriptVirtualizer` for wiring large transcript lists to a scroll container
|
||||||
|
|
||||||
## Install
|
## Install
|
||||||
|
|
@ -243,3 +244,47 @@ Useful `ChatComposer` props:
|
||||||
- `allowEmptySubmit` when the submit action is valid without draft text, such as a stop button
|
- `allowEmptySubmit` when the submit action is valid without draft text, such as a stop button
|
||||||
|
|
||||||
Use `transcriptProps` and `composerProps` when you want the shared composition but still need custom rendering or behavior. Use `transcriptClassNames` and `composerClassNames` when you want styling hooks for each subcomponent.
|
Use `transcriptProps` and `composerProps` when you want the shared composition but still need custom rendering or behavior. Use `transcriptClassNames` and `composerClassNames` when you want styling hooks for each subcomponent.
|
||||||
|
|
||||||
|
## Desktop viewer
|
||||||
|
|
||||||
|
`DesktopViewer` connects to a live desktop stream via WebRTC and renders the video feed with interactive mouse and keyboard input forwarding.
|
||||||
|
|
||||||
|
```tsx DesktopPane.tsx
|
||||||
|
"use client";
|
||||||
|
|
||||||
|
import { useEffect, useState } from "react";
|
||||||
|
import { SandboxAgent } from "sandbox-agent";
|
||||||
|
import { DesktopViewer } from "@sandbox-agent/react";
|
||||||
|
|
||||||
|
export default function DesktopPane() {
|
||||||
|
const [client, setClient] = useState<SandboxAgent | null>(null);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let sdk: SandboxAgent | null = null;
|
||||||
|
const start = async () => {
|
||||||
|
sdk = await SandboxAgent.connect({ baseUrl: "http://127.0.0.1:2468" });
|
||||||
|
await sdk.startDesktop();
|
||||||
|
await sdk.startDesktopStream();
|
||||||
|
setClient(sdk);
|
||||||
|
};
|
||||||
|
void start();
|
||||||
|
return () => { void sdk?.dispose(); };
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
if (!client) return <div>Starting desktop...</div>;
|
||||||
|
|
||||||
|
return <DesktopViewer client={client} height={600} />;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Props:
|
||||||
|
|
||||||
|
- `client`: a `SandboxAgent` client (or any object with `connectDesktopStream`)
|
||||||
|
- `height`, `style`, `imageStyle`: optional layout overrides
|
||||||
|
- `showStatusBar`: toggle the connection status bar (default `true`)
|
||||||
|
- `onConnect`, `onDisconnect`, `onError`: optional lifecycle callbacks
|
||||||
|
- `className` and `classNames`: external styling hooks
|
||||||
|
|
||||||
|
The component is unstyled by default. Use `classNames` slots (`root`, `statusBar`, `statusText`, `statusResolution`, `viewport`, `video`) and `data-slot`/`data-state` attributes for styling from outside the package.
|
||||||
|
|
||||||
|
See [Computer Use](/computer-use) for the lower-level desktop APIs.
|
||||||
|
|
|
||||||
|
|
@ -8,9 +8,19 @@ type ConnectionState = "connecting" | "ready" | "closed" | "error";
|
||||||
|
|
||||||
export type DesktopViewerClient = Pick<SandboxAgent, "connectDesktopStream">;
|
export type DesktopViewerClient = Pick<SandboxAgent, "connectDesktopStream">;
|
||||||
|
|
||||||
|
export interface DesktopViewerClassNames {
|
||||||
|
root?: string;
|
||||||
|
statusBar?: string;
|
||||||
|
statusText?: string;
|
||||||
|
statusResolution?: string;
|
||||||
|
viewport?: string;
|
||||||
|
video?: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface DesktopViewerProps {
|
export interface DesktopViewerProps {
|
||||||
client: DesktopViewerClient;
|
client: DesktopViewerClient;
|
||||||
className?: string;
|
className?: string;
|
||||||
|
classNames?: Partial<DesktopViewerClassNames>;
|
||||||
style?: CSSProperties;
|
style?: CSSProperties;
|
||||||
imageStyle?: CSSProperties;
|
imageStyle?: CSSProperties;
|
||||||
height?: number | string;
|
height?: number | string;
|
||||||
|
|
@ -20,66 +30,17 @@ export interface DesktopViewerProps {
|
||||||
onError?: (error: DesktopStreamErrorStatus | Error) => void;
|
onError?: (error: DesktopStreamErrorStatus | Error) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
const shellStyle: CSSProperties = {
|
const layoutStyles = {
|
||||||
display: "flex",
|
shell: { display: "flex", flexDirection: "column", overflow: "hidden" } as CSSProperties,
|
||||||
flexDirection: "column",
|
statusBar: { display: "flex", alignItems: "center", justifyContent: "space-between", gap: 12 } as CSSProperties,
|
||||||
overflow: "hidden",
|
viewport: { position: "relative", display: "flex", alignItems: "center", justifyContent: "center", overflow: "hidden" } as CSSProperties,
|
||||||
border: "1px solid rgba(15, 23, 42, 0.14)",
|
video: { display: "block", width: "100%", height: "100%", objectFit: "contain", userSelect: "none" } as CSSProperties,
|
||||||
borderRadius: 14,
|
|
||||||
background: "linear-gradient(180deg, rgba(248, 250, 252, 0.96) 0%, rgba(226, 232, 240, 0.92) 100%)",
|
|
||||||
boxShadow: "0 20px 40px rgba(15, 23, 42, 0.08)",
|
|
||||||
};
|
|
||||||
|
|
||||||
const statusBarStyle: CSSProperties = {
|
|
||||||
display: "flex",
|
|
||||||
alignItems: "center",
|
|
||||||
justifyContent: "space-between",
|
|
||||||
gap: 12,
|
|
||||||
padding: "10px 14px",
|
|
||||||
borderBottom: "1px solid rgba(15, 23, 42, 0.08)",
|
|
||||||
background: "rgba(255, 255, 255, 0.78)",
|
|
||||||
color: "#0f172a",
|
|
||||||
fontSize: 12,
|
|
||||||
lineHeight: 1.4,
|
|
||||||
};
|
|
||||||
|
|
||||||
const viewportStyle: CSSProperties = {
|
|
||||||
position: "relative",
|
|
||||||
display: "flex",
|
|
||||||
alignItems: "center",
|
|
||||||
justifyContent: "center",
|
|
||||||
overflow: "hidden",
|
|
||||||
background: "radial-gradient(circle at top, rgba(14, 165, 233, 0.18), transparent 45%), linear-gradient(180deg, #0f172a 0%, #111827 100%)",
|
|
||||||
};
|
|
||||||
|
|
||||||
const videoBaseStyle: CSSProperties = {
|
|
||||||
display: "block",
|
|
||||||
width: "100%",
|
|
||||||
height: "100%",
|
|
||||||
objectFit: "contain",
|
|
||||||
userSelect: "none",
|
|
||||||
};
|
|
||||||
|
|
||||||
const hintStyle: CSSProperties = {
|
|
||||||
opacity: 0.66,
|
|
||||||
};
|
|
||||||
|
|
||||||
const getStatusColor = (state: ConnectionState): string => {
|
|
||||||
switch (state) {
|
|
||||||
case "ready":
|
|
||||||
return "#15803d";
|
|
||||||
case "error":
|
|
||||||
return "#b91c1c";
|
|
||||||
case "closed":
|
|
||||||
return "#b45309";
|
|
||||||
default:
|
|
||||||
return "#475569";
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
export const DesktopViewer = ({
|
export const DesktopViewer = ({
|
||||||
client,
|
client,
|
||||||
className,
|
className,
|
||||||
|
classNames,
|
||||||
style,
|
style,
|
||||||
imageStyle,
|
imageStyle,
|
||||||
height = 480,
|
height = 480,
|
||||||
|
|
@ -91,11 +52,18 @@ export const DesktopViewer = ({
|
||||||
const wrapperRef = useRef<HTMLDivElement | null>(null);
|
const wrapperRef = useRef<HTMLDivElement | null>(null);
|
||||||
const videoRef = useRef<HTMLVideoElement | null>(null);
|
const videoRef = useRef<HTMLVideoElement | null>(null);
|
||||||
const sessionRef = useRef<ReturnType<DesktopViewerClient["connectDesktopStream"]> | null>(null);
|
const sessionRef = useRef<ReturnType<DesktopViewerClient["connectDesktopStream"]> | null>(null);
|
||||||
|
const onConnectRef = useRef(onConnect);
|
||||||
|
const onDisconnectRef = useRef(onDisconnect);
|
||||||
|
const onErrorRef = useRef(onError);
|
||||||
const [connectionState, setConnectionState] = useState<ConnectionState>("connecting");
|
const [connectionState, setConnectionState] = useState<ConnectionState>("connecting");
|
||||||
const [statusMessage, setStatusMessage] = useState("Starting desktop stream...");
|
const [statusMessage, setStatusMessage] = useState("Starting desktop stream...");
|
||||||
const [hasVideo, setHasVideo] = useState(false);
|
const [hasVideo, setHasVideo] = useState(false);
|
||||||
const [resolution, setResolution] = useState<{ width: number; height: number } | null>(null);
|
const [resolution, setResolution] = useState<{ width: number; height: number } | null>(null);
|
||||||
|
|
||||||
|
onConnectRef.current = onConnect;
|
||||||
|
onDisconnectRef.current = onDisconnect;
|
||||||
|
onErrorRef.current = onError;
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
let cancelled = false;
|
let cancelled = false;
|
||||||
|
|
||||||
|
|
@ -112,7 +80,7 @@ export const DesktopViewer = ({
|
||||||
setConnectionState("ready");
|
setConnectionState("ready");
|
||||||
setStatusMessage("Desktop stream connected.");
|
setStatusMessage("Desktop stream connected.");
|
||||||
setResolution({ width: status.width, height: status.height });
|
setResolution({ width: status.width, height: status.height });
|
||||||
onConnect?.(status);
|
onConnectRef.current?.(status);
|
||||||
});
|
});
|
||||||
session.onTrack((stream) => {
|
session.onTrack((stream) => {
|
||||||
if (cancelled) return;
|
if (cancelled) return;
|
||||||
|
|
@ -127,13 +95,13 @@ export const DesktopViewer = ({
|
||||||
if (cancelled) return;
|
if (cancelled) return;
|
||||||
setConnectionState("error");
|
setConnectionState("error");
|
||||||
setStatusMessage(error instanceof Error ? error.message : error.message);
|
setStatusMessage(error instanceof Error ? error.message : error.message);
|
||||||
onError?.(error);
|
onErrorRef.current?.(error);
|
||||||
});
|
});
|
||||||
session.onDisconnect(() => {
|
session.onDisconnect(() => {
|
||||||
if (cancelled) return;
|
if (cancelled) return;
|
||||||
setConnectionState((current) => (current === "error" ? current : "closed"));
|
setConnectionState((current) => (current === "error" ? current : "closed"));
|
||||||
setStatusMessage((current) => (current === "Desktop stream connected." ? "Desktop stream disconnected." : current));
|
setStatusMessage((current) => (current === "Desktop stream connected." ? "Desktop stream disconnected." : current));
|
||||||
onDisconnect?.();
|
onDisconnectRef.current?.();
|
||||||
});
|
});
|
||||||
|
|
||||||
return () => {
|
return () => {
|
||||||
|
|
@ -146,7 +114,7 @@ export const DesktopViewer = ({
|
||||||
}
|
}
|
||||||
setHasVideo(false);
|
setHasVideo(false);
|
||||||
};
|
};
|
||||||
}, [client, onConnect, onDisconnect, onError]);
|
}, [client]);
|
||||||
|
|
||||||
const scalePoint = (clientX: number, clientY: number) => {
|
const scalePoint = (clientX: number, clientY: number) => {
|
||||||
const video = videoRef.current;
|
const video = videoRef.current;
|
||||||
|
|
@ -204,18 +172,24 @@ export const DesktopViewer = ({
|
||||||
};
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className={className} style={{ ...shellStyle, ...style }}>
|
<div className={classNames?.root ?? className} data-slot="root" data-state={connectionState} style={{ ...layoutStyles.shell, ...style }}>
|
||||||
{showStatusBar ? (
|
{showStatusBar ? (
|
||||||
<div style={statusBarStyle}>
|
<div className={classNames?.statusBar} data-slot="status-bar" style={layoutStyles.statusBar}>
|
||||||
<span style={{ color: getStatusColor(connectionState) }}>{statusMessage}</span>
|
<span className={classNames?.statusText} data-slot="status-text" data-state={connectionState}>
|
||||||
<span style={hintStyle}>{resolution ? `${resolution.width}×${resolution.height}` : "Awaiting stream"}</span>
|
{statusMessage}
|
||||||
|
</span>
|
||||||
|
<span className={classNames?.statusResolution} data-slot="status-resolution">
|
||||||
|
{resolution ? `${resolution.width}×${resolution.height}` : "Awaiting stream"}
|
||||||
|
</span>
|
||||||
</div>
|
</div>
|
||||||
) : null}
|
) : null}
|
||||||
<div
|
<div
|
||||||
ref={wrapperRef}
|
ref={wrapperRef}
|
||||||
|
className={classNames?.viewport}
|
||||||
|
data-slot="viewport"
|
||||||
role="button"
|
role="button"
|
||||||
tabIndex={0}
|
tabIndex={0}
|
||||||
style={{ ...viewportStyle, height }}
|
style={{ ...layoutStyles.viewport, height }}
|
||||||
onMouseMove={(event) => {
|
onMouseMove={(event) => {
|
||||||
const point = scalePoint(event.clientX, event.clientY);
|
const point = scalePoint(event.clientX, event.clientY);
|
||||||
if (!point) {
|
if (!point) {
|
||||||
|
|
@ -259,12 +233,14 @@ export const DesktopViewer = ({
|
||||||
>
|
>
|
||||||
<video
|
<video
|
||||||
ref={videoRef}
|
ref={videoRef}
|
||||||
|
className={classNames?.video}
|
||||||
|
data-slot="video"
|
||||||
autoPlay
|
autoPlay
|
||||||
playsInline
|
playsInline
|
||||||
muted
|
muted
|
||||||
tabIndex={-1}
|
tabIndex={-1}
|
||||||
draggable={false}
|
draggable={false}
|
||||||
style={{ ...videoBaseStyle, ...imageStyle, display: hasVideo ? "block" : "none", pointerEvents: "none" }}
|
style={{ ...layoutStyles.video, ...imageStyle, display: hasVideo ? "block" : "none", pointerEvents: "none" }}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,7 @@ export type {
|
||||||
} from "./ChatComposer.tsx";
|
} from "./ChatComposer.tsx";
|
||||||
|
|
||||||
export type {
|
export type {
|
||||||
|
DesktopViewerClassNames,
|
||||||
DesktopViewerClient,
|
DesktopViewerClient,
|
||||||
DesktopViewerProps,
|
DesktopViewerProps,
|
||||||
} from "./DesktopViewer.tsx";
|
} from "./DesktopViewer.tsx";
|
||||||
|
|
|
||||||
|
|
@ -2005,7 +2005,7 @@ export class SandboxAgent {
|
||||||
}
|
}
|
||||||
|
|
||||||
connectDesktopStream(options: DesktopStreamSessionOptions = {}): DesktopStreamSession {
|
connectDesktopStream(options: DesktopStreamSessionOptions = {}): DesktopStreamSession {
|
||||||
return new DesktopStreamSession(this.connectDesktopStreamWebSocket(options));
|
return new DesktopStreamSession(this.connectDesktopStreamWebSocket(options), options);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async getLiveConnection(agent: string): Promise<LiveAcpConnection> {
|
private async getLiveConnection(agent: string): Promise<LiveAcpConnection> {
|
||||||
|
|
|
||||||
|
|
@ -64,17 +64,13 @@ impl DesktopRecordingManager {
|
||||||
|
|
||||||
self.ensure_recordings_dir()?;
|
self.ensure_recordings_dir()?;
|
||||||
|
|
||||||
{
|
|
||||||
let mut state = self.inner.lock().await;
|
|
||||||
self.refresh_locked(&mut state).await?;
|
|
||||||
if state.current_id.is_some() {
|
|
||||||
return Err(SandboxError::Conflict {
|
|
||||||
message: "a desktop recording is already active".to_string(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut state = self.inner.lock().await;
|
let mut state = self.inner.lock().await;
|
||||||
|
self.refresh_locked(&mut state).await?;
|
||||||
|
if state.current_id.is_some() {
|
||||||
|
return Err(SandboxError::Conflict {
|
||||||
|
message: "a desktop recording is already active".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
let id_num = state.next_id + 1;
|
let id_num = state.next_id + 1;
|
||||||
state.next_id = id_num;
|
state.next_id = id_num;
|
||||||
let id = format!("rec_{id_num}");
|
let id = format!("rec_{id_num}");
|
||||||
|
|
|
||||||
|
|
@ -2036,14 +2036,18 @@ impl DesktopRuntime {
|
||||||
options: &DesktopScreenshotOptions,
|
options: &DesktopScreenshotOptions,
|
||||||
region_x: i32,
|
region_x: i32,
|
||||||
region_y: i32,
|
region_y: i32,
|
||||||
_region_width: u32,
|
region_width: u32,
|
||||||
_region_height: u32,
|
region_height: u32,
|
||||||
) -> Result<Vec<u8>, DesktopProblem> {
|
) -> Result<Vec<u8>, DesktopProblem> {
|
||||||
let pos = self.mouse_position_locked(state, ready).await?;
|
let pos = self.mouse_position_locked(state, ready).await?;
|
||||||
// Adjust cursor position relative to the region
|
// Adjust cursor position relative to the region
|
||||||
let cursor_x = pos.x - region_x;
|
let cursor_x = pos.x - region_x;
|
||||||
let cursor_y = pos.y - region_y;
|
let cursor_y = pos.y - region_y;
|
||||||
if cursor_x < 0 || cursor_y < 0 {
|
if cursor_x < 0
|
||||||
|
|| cursor_y < 0
|
||||||
|
|| cursor_x >= region_width as i32
|
||||||
|
|| cursor_y >= region_height as i32
|
||||||
|
{
|
||||||
// Cursor is outside the region, return screenshot as-is
|
// Cursor is outside the region, return screenshot as-is
|
||||||
return Ok(screenshot_bytes);
|
return Ok(screenshot_bytes);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue