harivansh-afk/sandbox-agent
1
0
Fork 0
mirror of https://github.com/harivansh-afk/sandbox-agent.git synced 2026-04-15 09:01:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: desktop computer-use APIs with windows, launch/open, and neko streaming

Browse source 
Adds desktop computer-use endpoints (windows, screenshots, mouse/keyboard,
launch/open), enhances neko-based streaming integration, updates inspector
UI with desktop debug tab, and adds common software test infrastructure.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Nathan Flurry 2026-03-17 02:35:52 -07:00
parent 2d8508d6e2
commit dff7614b11
17 changed files with 4045 additions and 136 deletions
Download patch file Download diff file Expand all files Collapse all files

23
sdks/react/src/DesktopViewer.tsx
View file

@ -14,6 +14,7 @@ export interface DesktopViewerProps {
style?: CSSProperties;
imageStyle?: CSSProperties;
height?: number | string;
showStatusBar?: boolean;
onConnect?: (status: DesktopStreamReadyStatus) => void;
onDisconnect?: () => void;
onError?: (error: DesktopStreamErrorStatus | Error) => void;
@ -76,7 +77,17 @@ const getStatusColor = (state: ConnectionState): string => {
}
};
export const DesktopViewer = ({ client, className, style, imageStyle, height = 480, onConnect, onDisconnect, onError }: DesktopViewerProps) => {
export const DesktopViewer = ({
client,
className,
style,
imageStyle,
height = 480,
showStatusBar = true,
onConnect,
onDisconnect,
onError,
}: DesktopViewerProps) => {
const wrapperRef = useRef<HTMLDivElement | null>(null);
const videoRef = useRef<HTMLVideoElement | null>(null);
const sessionRef = useRef<ReturnType<DesktopViewerClient["connectDesktopStream"]> | null>(null);
@ -194,10 +205,12 @@ export const DesktopViewer = ({ client, className, style, imageStyle, height = 4
return (
<div className={className} style={{ ...shellStyle, ...style }}>
<div style={statusBarStyle}>
<span style={{ color: getStatusColor(connectionState) }}>{statusMessage}</span>
<span style={hintStyle}>{resolution ? `${resolution.width}×${resolution.height}` : "Awaiting stream"}</span>
</div>
{showStatusBar ? (
<div style={statusBarStyle}>
<span style={{ color: getStatusColor(connectionState) }}>{statusMessage}</span>
<span style={hintStyle}>{resolution ? `${resolution.width}×${resolution.height}` : "Awaiting stream"}</span>
</div>
) : null}
<div
ref={wrapperRef}
role="button"

58
sdks/typescript/src/client.ts
View file

@ -31,10 +31,15 @@ import {
type AgentInstallResponse,
type AgentListResponse,
type DesktopActionResponse,
type DesktopClipboardQuery,
type DesktopClipboardResponse,
type DesktopClipboardWriteRequest,
type DesktopDisplayInfoResponse,
type DesktopKeyboardDownRequest,
type DesktopKeyboardPressRequest,
type DesktopKeyboardTypeRequest,
type DesktopLaunchRequest,
type DesktopLaunchResponse,
type DesktopMouseClickRequest,
type DesktopMouseDownRequest,
type DesktopMouseDragRequest,
@ -43,6 +48,8 @@ import {
type DesktopMouseScrollRequest,
type DesktopMouseUpRequest,
type DesktopKeyboardUpRequest,
type DesktopOpenRequest,
type DesktopOpenResponse,
type DesktopRecordingInfo,
type DesktopRecordingListResponse,
type DesktopRecordingStartRequest,
@ -51,7 +58,10 @@ import {
type DesktopStartRequest,
type DesktopStatusResponse,
type DesktopStreamStatusResponse,
type DesktopWindowInfo,
type DesktopWindowListResponse,
type DesktopWindowMoveRequest,
type DesktopWindowResizeRequest,
type FsActionResponse,
type FsDeleteQuery,
type FsEntriesQuery,
@ -1663,6 +1673,54 @@ export class SandboxAgent {
return this.requestJson("GET", `${API_PREFIX}/desktop/windows`);
}
async getDesktopFocusedWindow(): Promise<DesktopWindowInfo> {
return this.requestJson("GET", `${API_PREFIX}/desktop/windows/focused`);
}
async focusDesktopWindow(windowId: string): Promise<DesktopWindowInfo> {
return this.requestJson("POST", `${API_PREFIX}/desktop/windows/${encodeURIComponent(windowId)}/focus`);
}
async moveDesktopWindow(windowId: string, request: DesktopWindowMoveRequest): Promise<DesktopWindowInfo> {
return this.requestJson("POST", `${API_PREFIX}/desktop/windows/${encodeURIComponent(windowId)}/move`, {
body: request,
});
}
async resizeDesktopWindow(windowId: string, request: DesktopWindowResizeRequest): Promise<DesktopWindowInfo> {
return this.requestJson("POST", `${API_PREFIX}/desktop/windows/${encodeURIComponent(windowId)}/resize`, {
body: request,
});
}
async getDesktopClipboard(query: DesktopClipboardQuery = {}): Promise<DesktopClipboardResponse> {
return this.requestJson("GET", `${API_PREFIX}/desktop/clipboard`, {
query,
});
}
async setDesktopClipboard(request: DesktopClipboardWriteRequest): Promise<DesktopActionResponse> {
return this.requestJson("POST", `${API_PREFIX}/desktop/clipboard`, {
body: request,
});
}
async launchDesktopApp(request: DesktopLaunchRequest): Promise<DesktopLaunchResponse> {
return this.requestJson("POST", `${API_PREFIX}/desktop/launch`, {
body: request,
});
}
async openDesktopTarget(request: DesktopOpenRequest): Promise<DesktopOpenResponse> {
return this.requestJson("POST", `${API_PREFIX}/desktop/open`, {
body: request,
});
}
async getDesktopStreamStatus(): Promise<DesktopStreamStatusResponse> {
return this.requestJson("GET", `${API_PREFIX}/desktop/stream/status`);
}
async startDesktopRecording(request: DesktopRecordingStartRequest = {}): Promise<DesktopRecordingInfo> {
return this.requestJson("POST", `${API_PREFIX}/desktop/recording/start`, {
body: request,

467
sdks/typescript/src/generated/openapi.ts
View file

@ -31,6 +31,18 @@ export interface paths {
put: operations["put_v1_config_skills"];
delete: operations["delete_v1_config_skills"];
};
"/v1/desktop/clipboard": {
/**
* Read the desktop clipboard.
* @description Returns the current text content of the X11 clipboard.
*/
get: operations["get_v1_desktop_clipboard"];
/**
* Write to the desktop clipboard.
* @description Sets the text content of the X11 clipboard.
*/
post: operations["post_v1_desktop_clipboard"];
};
"/v1/desktop/display/info": {
/**
* Get desktop display information.
@ -71,6 +83,14 @@ export interface paths {
*/
post: operations["post_v1_desktop_keyboard_up"];
};
"/v1/desktop/launch": {
/**
* Launch a desktop application.
* @description Launches an application by name on the managed desktop, optionally waiting
* for its window to appear.
*/
post: operations["post_v1_desktop_launch"];
};
"/v1/desktop/mouse/click": {
/**
* Click on the desktop.
@ -126,6 +146,13 @@ export interface paths {
*/
post: operations["post_v1_desktop_mouse_up"];
};
"/v1/desktop/open": {
/**
* Open a file or URL with the default handler.
* @description Opens a file path or URL using xdg-open on the managed desktop.
*/
post: operations["post_v1_desktop_open"];
};
"/v1/desktop/recording/start": {
/**
* Start desktop recording.
@ -208,6 +235,15 @@ export interface paths {
*/
post: operations["post_v1_desktop_stop"];
};
"/v1/desktop/stream/signaling": {
/**
* Open a desktop WebRTC signaling session.
* @description Upgrades the connection to a WebSocket used for WebRTC signaling between
* the browser client and the desktop streaming process. Also accepts mouse
* and keyboard input frames as a fallback transport.
*/
get: operations["get_v1_desktop_stream_ws"];
};
"/v1/desktop/stream/start": {
/**
* Start desktop streaming.
@ -215,6 +251,13 @@ export interface paths {
*/
post: operations["post_v1_desktop_stream_start"];
};
"/v1/desktop/stream/status": {
/**
* Get desktop stream status.
* @description Returns the current state of the desktop WebRTC streaming session.
*/
get: operations["get_v1_desktop_stream_status"];
};
"/v1/desktop/stream/stop": {
/**
* Stop desktop streaming.
@ -222,14 +265,6 @@ export interface paths {
*/
post: operations["post_v1_desktop_stream_stop"];
};
"/v1/desktop/stream/ws": {
/**
* Open a desktop websocket streaming session.
* @description Upgrades the connection to a websocket that streams JPEG desktop frames and
* accepts mouse and keyboard control frames.
*/
get: operations["get_v1_desktop_stream_ws"];
};
"/v1/desktop/windows": {
/**
* List visible desktop windows.
@ -238,6 +273,34 @@ export interface paths {
*/
get: operations["get_v1_desktop_windows"];
};
"/v1/desktop/windows/focused": {
/**
* Get the currently focused desktop window.
* @description Returns information about the window that currently has input focus.
*/
get: operations["get_v1_desktop_windows_focused"];
};
"/v1/desktop/windows/{id}/focus": {
/**
* Focus a desktop window.
* @description Brings the specified window to the foreground and gives it input focus.
*/
post: operations["post_v1_desktop_window_focus"];
};
"/v1/desktop/windows/{id}/move": {
/**
* Move a desktop window.
* @description Moves the specified window to the given position.
*/
post: operations["post_v1_desktop_window_move"];
};
"/v1/desktop/windows/{id}/resize": {
/**
* Resize a desktop window.
* @description Resizes the specified window to the given dimensions.
*/
post: operations["post_v1_desktop_window_resize"];
};
"/v1/fs/entries": {
get: operations["get_v1_fs_entries"];
};
@ -443,6 +506,17 @@ export interface components {
DesktopActionResponse: {
ok: boolean;
};
DesktopClipboardQuery: {
selection?: string | null;
};
DesktopClipboardResponse: {
selection: string;
text: string;
};
DesktopClipboardWriteRequest: {
selection?: string | null;
text: string;
};
DesktopDisplayInfoResponse: {
display: string;
resolution: components["schemas"]["DesktopResolution"];
@ -472,6 +546,17 @@ export interface components {
DesktopKeyboardUpRequest: {
key: string;
};
DesktopLaunchRequest: {
app: string;
args?: string[] | null;
wait?: boolean | null;
};
DesktopLaunchResponse: {
/** Format: int32 */
pid?: number | null;
processId: string;
windowId?: string | null;
};
/** @enum {string} */
DesktopMouseButton: "left" | "middle" | "right";
DesktopMouseClickRequest: {
@ -533,6 +618,14 @@ export interface components {
/** Format: int32 */
y?: number | null;
};
DesktopOpenRequest: {
target: string;
};
DesktopOpenResponse: {
/** Format: int32 */
pid?: number | null;
processId: string;
};
DesktopProcessInfo: {
logPath?: string | null;
name: string;
@ -567,6 +660,7 @@ export interface components {
quality?: number | null;
/** Format: float */
scale?: number | null;
showCursor?: boolean | null;
/** Format: int32 */
width: number;
/** Format: int32 */
@ -590,13 +684,24 @@ export interface components {
quality?: number | null;
/** Format: float */
scale?: number | null;
showCursor?: boolean | null;
};
DesktopStartRequest: {
/** Format: int32 */
displayNum?: number | null;
/** Format: int32 */
dpi?: number | null;
/** Format: int32 */
height?: number | null;
/** Format: int32 */
recordingFps?: number | null;
stateDir?: string | null;
streamAudioCodec?: string | null;
/** Format: int32 */
streamFrameRate?: number | null;
streamVideoCodec?: string | null;
webrtcPortRange?: string | null;
/** Format: int32 */
width?: number | null;
};
/** @enum {string} */
@ -611,9 +716,13 @@ export interface components {
runtimeLogPath?: string | null;
startedAt?: string | null;
state: components["schemas"]["DesktopState"];
/** @description Current visible windows (included when the desktop is active). */
windows?: components["schemas"]["DesktopWindowInfo"][];
};
DesktopStreamStatusResponse: {
active: boolean;
processId?: string | null;
windowId?: string | null;
};
DesktopWindowInfo: {
/** Format: int32 */
@ -631,6 +740,18 @@ export interface components {
DesktopWindowListResponse: {
windows: components["schemas"]["DesktopWindowInfo"][];
};
DesktopWindowMoveRequest: {
/** Format: int32 */
x: number;
/** Format: int32 */
y: number;
};
DesktopWindowResizeRequest: {
/** Format: int32 */
height: number;
/** Format: int32 */
width: number;
};
/** @enum {string} */
ErrorType:
| "invalid_request"
@ -1231,6 +1352,68 @@ export interface operations {
};
};
};
/**
* Read the desktop clipboard.
* @description Returns the current text content of the X11 clipboard.
*/
get_v1_desktop_clipboard: {
parameters: {
query?: {
selection?: string | null;
};
};
responses: {
/** @description Clipboard contents */
200: {
content: {
"application/json": components["schemas"]["DesktopClipboardResponse"];
};
};
/** @description Desktop runtime is not ready */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Clipboard read failed */
500: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
/**
* Write to the desktop clipboard.
* @description Sets the text content of the X11 clipboard.
*/
post_v1_desktop_clipboard: {
requestBody: {
content: {
"application/json": components["schemas"]["DesktopClipboardWriteRequest"];
};
};
responses: {
/** @description Clipboard updated */
200: {
content: {
"application/json": components["schemas"]["DesktopActionResponse"];
};
};
/** @description Desktop runtime is not ready */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Clipboard write failed */
500: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
/**
* Get desktop display information.
* @description Performs a health-gated display query against the managed desktop and
@ -1410,6 +1593,38 @@ export interface operations {
};
};
};
/**
* Launch a desktop application.
* @description Launches an application by name on the managed desktop, optionally waiting
* for its window to appear.
*/
post_v1_desktop_launch: {
requestBody: {
content: {
"application/json": components["schemas"]["DesktopLaunchRequest"];
};
};
responses: {
/** @description Application launched */
200: {
content: {
"application/json": components["schemas"]["DesktopLaunchResponse"];
};
};
/** @description Application not found */
404: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime is not ready */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
/**
* Click on the desktop.
* @description Performs a health-gated pointer move and click against the managed desktop
@ -1664,6 +1879,31 @@ export interface operations {
};
};
};
/**
* Open a file or URL with the default handler.
* @description Opens a file path or URL using xdg-open on the managed desktop.
*/
post_v1_desktop_open: {
requestBody: {
content: {
"application/json": components["schemas"]["DesktopOpenRequest"];
};
};
responses: {
/** @description Target opened */
200: {
content: {
"application/json": components["schemas"]["DesktopOpenResponse"];
};
};
/** @description Desktop runtime is not ready */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
/**
* Start desktop recording.
* @description Starts an ffmpeg x11grab recording against the managed desktop and returns
@ -1834,6 +2074,7 @@ export interface operations {
format?: components["schemas"]["DesktopScreenshotFormat"] | null;
quality?: number | null;
scale?: number | null;
showCursor?: boolean | null;
};
};
responses: {
@ -1876,6 +2117,7 @@ export interface operations {
format?: components["schemas"]["DesktopScreenshotFormat"] | null;
quality?: number | null;
scale?: number | null;
showCursor?: boolean | null;
};
};
responses: {
@ -1990,37 +2232,10 @@ export interface operations {
};
};
/**
* Start desktop streaming.
* @description Enables desktop websocket streaming for the managed desktop.
*/
post_v1_desktop_stream_start: {
responses: {
/** @description Desktop streaming started */
200: {
content: {
"application/json": components["schemas"]["DesktopStreamStatusResponse"];
};
};
};
};
/**
* Stop desktop streaming.
* @description Disables desktop websocket streaming for the managed desktop.
*/
post_v1_desktop_stream_stop: {
responses: {
/** @description Desktop streaming stopped */
200: {
content: {
"application/json": components["schemas"]["DesktopStreamStatusResponse"];
};
};
};
};
/**
* Open a desktop websocket streaming session.
* @description Upgrades the connection to a websocket that streams JPEG desktop frames and
* accepts mouse and keyboard control frames.
* Open a desktop WebRTC signaling session.
* @description Upgrades the connection to a WebSocket used for WebRTC signaling between
* the browser client and the desktop streaming process. Also accepts mouse
* and keyboard input frames as a fallback transport.
*/
get_v1_desktop_stream_ws: {
parameters: {
@ -2048,6 +2263,48 @@ export interface operations {
};
};
};
/**
* Start desktop streaming.
* @description Enables desktop websocket streaming for the managed desktop.
*/
post_v1_desktop_stream_start: {
responses: {
/** @description Desktop streaming started */
200: {
content: {
"application/json": components["schemas"]["DesktopStreamStatusResponse"];
};
};
};
};
/**
* Get desktop stream status.
* @description Returns the current state of the desktop WebRTC streaming session.
*/
get_v1_desktop_stream_status: {
responses: {
/** @description Desktop stream status */
200: {
content: {
"application/json": components["schemas"]["DesktopStreamStatusResponse"];
};
};
};
};
/**
* Stop desktop streaming.
* @description Disables desktop websocket streaming for the managed desktop.
*/
post_v1_desktop_stream_stop: {
responses: {
/** @description Desktop streaming stopped */
200: {
content: {
"application/json": components["schemas"]["DesktopStreamStatusResponse"];
};
};
};
};
/**
* List visible desktop windows.
* @description Performs a health-gated visible-window enumeration against the managed
@ -2075,6 +2332,138 @@ export interface operations {
};
};
};
/**
* Get the currently focused desktop window.
* @description Returns information about the window that currently has input focus.
*/
get_v1_desktop_windows_focused: {
responses: {
/** @description Focused window info */
200: {
content: {
"application/json": components["schemas"]["DesktopWindowInfo"];
};
};
/** @description No window is focused */
404: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime is not ready */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
/**
* Focus a desktop window.
* @description Brings the specified window to the foreground and gives it input focus.
*/
post_v1_desktop_window_focus: {
parameters: {
path: {
/** @description X11 window ID */
id: string;
};
};
responses: {
/** @description Window info after focus */
200: {
content: {
"application/json": components["schemas"]["DesktopWindowInfo"];
};
};
/** @description Window not found */
404: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime is not ready */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
/**
* Move a desktop window.
* @description Moves the specified window to the given position.
*/
post_v1_desktop_window_move: {
parameters: {
path: {
/** @description X11 window ID */
id: string;
};
};
requestBody: {
content: {
"application/json": components["schemas"]["DesktopWindowMoveRequest"];
};
};
responses: {
/** @description Window info after move */
200: {
content: {
"application/json": components["schemas"]["DesktopWindowInfo"];
};
};
/** @description Window not found */
404: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime is not ready */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
/**
* Resize a desktop window.
* @description Resizes the specified window to the given dimensions.
*/
post_v1_desktop_window_resize: {
parameters: {
path: {
/** @description X11 window ID */
id: string;
};
};
requestBody: {
content: {
"application/json": components["schemas"]["DesktopWindowResizeRequest"];
};
};
responses: {
/** @description Window info after resize */
200: {
content: {
"application/json": components["schemas"]["DesktopWindowInfo"];
};
};
/** @description Window not found */
404: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime is not ready */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
get_v1_fs_entries: {
parameters: {
query?: {

10
sdks/typescript/src/types.ts
View file

@ -36,6 +36,16 @@ export type DesktopRecordingStatus = components["schemas"]["DesktopRecordingStat
export type DesktopRecordingInfo = JsonResponse<operations["post_v1_desktop_recording_start"], 200>;
export type DesktopRecordingListResponse = JsonResponse<operations["get_v1_desktop_recordings"], 200>;
export type DesktopStreamStatusResponse = JsonResponse<operations["post_v1_desktop_stream_start"], 200>;
export type DesktopClipboardResponse = JsonResponse<operations["get_v1_desktop_clipboard"], 200>;
export type DesktopClipboardQuery =
QueryParams<operations["get_v1_desktop_clipboard"]> extends never ? Record<string, never> : QueryParams<operations["get_v1_desktop_clipboard"]>;
export type DesktopClipboardWriteRequest = JsonRequestBody<operations["post_v1_desktop_clipboard"]>;
export type DesktopLaunchRequest = JsonRequestBody<operations["post_v1_desktop_launch"]>;
export type DesktopLaunchResponse = JsonResponse<operations["post_v1_desktop_launch"], 200>;
export type DesktopOpenRequest = JsonRequestBody<operations["post_v1_desktop_open"]>;
export type DesktopOpenResponse = JsonResponse<operations["post_v1_desktop_open"], 200>;
export type DesktopWindowMoveRequest = JsonRequestBody<operations["post_v1_desktop_window_move"]>;
export type DesktopWindowResizeRequest = JsonRequestBody<operations["post_v1_desktop_window_resize"]>;
export type AgentListResponse = JsonResponse<operations["get_v1_agents"], 200>;
export type AgentInfo = components["schemas"]["AgentInfo"];
export type AgentQuery = QueryParams<operations["get_v1_agents"]>;