harivansh-afk/sandbox-agent
1
0
Fork 0
mirror of https://github.com/harivansh-afk/sandbox-agent.git synced 2026-04-16 22:03:52 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add desktop runtime API and SDK support

Browse source
This commit is contained in:
Nathan Flurry 2026-03-07 23:32:49 -08:00
parent 3d9476ed0b
commit 641597afe6
27 changed files with 5881 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

89
sdks/typescript/src/client.ts
View file

@ -25,6 +25,19 @@ import {
type AgentInstallRequest,
type AgentInstallResponse,
type AgentListResponse,
type DesktopActionResponse,
type DesktopDisplayInfoResponse,
type DesktopKeyboardPressRequest,
type DesktopKeyboardTypeRequest,
type DesktopMouseClickRequest,
type DesktopMouseDragRequest,
type DesktopMouseMoveRequest,
type DesktopMousePositionResponse,
type DesktopMouseScrollRequest,
type DesktopRegionScreenshotQuery,
type DesktopScreenshotQuery,
type DesktopStartRequest,
type DesktopStatusResponse,
type FsActionResponse,
type FsDeleteQuery,
type FsEntriesQuery,
@ -1294,6 +1307,82 @@ export class SandboxAgent {
return this.requestHealth();
}
async startDesktop(request: DesktopStartRequest = {}): Promise<DesktopStatusResponse> {
return this.requestJson("POST", `${API_PREFIX}/desktop/start`, {
body: request,
});
}
async stopDesktop(): Promise<DesktopStatusResponse> {
return this.requestJson("POST", `${API_PREFIX}/desktop/stop`);
}
async getDesktopStatus(): Promise<DesktopStatusResponse> {
return this.requestJson("GET", `${API_PREFIX}/desktop/status`);
}
async getDesktopDisplayInfo(): Promise<DesktopDisplayInfoResponse> {
return this.requestJson("GET", `${API_PREFIX}/desktop/display/info`);
}
async takeDesktopScreenshot(query: DesktopScreenshotQuery = {}): Promise<Uint8Array> {
const response = await this.requestRaw("GET", `${API_PREFIX}/desktop/screenshot`, {
query,
accept: "image/png",
});
const buffer = await response.arrayBuffer();
return new Uint8Array(buffer);
}
async takeDesktopRegionScreenshot(query: DesktopRegionScreenshotQuery): Promise<Uint8Array> {
const response = await this.requestRaw("GET", `${API_PREFIX}/desktop/screenshot/region`, {
query,
accept: "image/png",
});
const buffer = await response.arrayBuffer();
return new Uint8Array(buffer);
}
async getDesktopMousePosition(): Promise<DesktopMousePositionResponse> {
return this.requestJson("GET", `${API_PREFIX}/desktop/mouse/position`);
}
async moveDesktopMouse(request: DesktopMouseMoveRequest): Promise<DesktopMousePositionResponse> {
return this.requestJson("POST", `${API_PREFIX}/desktop/mouse/move`, {
body: request,
});
}
async clickDesktop(request: DesktopMouseClickRequest): Promise<DesktopMousePositionResponse> {
return this.requestJson("POST", `${API_PREFIX}/desktop/mouse/click`, {
body: request,
});
}
async dragDesktopMouse(request: DesktopMouseDragRequest): Promise<DesktopMousePositionResponse> {
return this.requestJson("POST", `${API_PREFIX}/desktop/mouse/drag`, {
body: request,
});
}
async scrollDesktop(request: DesktopMouseScrollRequest): Promise<DesktopMousePositionResponse> {
return this.requestJson("POST", `${API_PREFIX}/desktop/mouse/scroll`, {
body: request,
});
}
async typeDesktopText(request: DesktopKeyboardTypeRequest): Promise<DesktopActionResponse> {
return this.requestJson("POST", `${API_PREFIX}/desktop/keyboard/type`, {
body: request,
});
}
async pressDesktopKey(request: DesktopKeyboardPressRequest): Promise<DesktopActionResponse> {
return this.requestJson("POST", `${API_PREFIX}/desktop/keyboard/press`, {
body: request,
});
}
async listAgents(options?: AgentQueryOptions): Promise<AgentListResponse> {
return this.requestJson("GET", `${API_PREFIX}/agents`, {
query: toAgentQuery(options),

651
sdks/typescript/src/generated/openapi.ts
View file

@ -32,6 +32,109 @@ export interface paths {
put: operations["put_v1_config_skills"];
delete: operations["delete_v1_config_skills"];
};
"/v1/desktop/display/info": {
/**
* Get desktop display information.
* @description Performs a health-gated display query against the managed desktop and
* returns the current display identifier and resolution.
*/
get: operations["get_v1_desktop_display_info"];
};
"/v1/desktop/keyboard/press": {
/**
* Press a desktop keyboard shortcut.
* @description Performs a health-gated `xdotool key` operation against the managed
* desktop.
*/
post: operations["post_v1_desktop_keyboard_press"];
};
"/v1/desktop/keyboard/type": {
/**
* Type desktop keyboard text.
* @description Performs a health-gated `xdotool type` operation against the managed
* desktop.
*/
post: operations["post_v1_desktop_keyboard_type"];
};
"/v1/desktop/mouse/click": {
/**
* Click on the desktop.
* @description Performs a health-gated pointer move and click against the managed desktop
* and returns the resulting mouse position.
*/
post: operations["post_v1_desktop_mouse_click"];
};
"/v1/desktop/mouse/drag": {
/**
* Drag the desktop mouse.
* @description Performs a health-gated drag gesture against the managed desktop and
* returns the resulting mouse position.
*/
post: operations["post_v1_desktop_mouse_drag"];
};
"/v1/desktop/mouse/move": {
/**
* Move the desktop mouse.
* @description Performs a health-gated absolute pointer move on the managed desktop and
* returns the resulting mouse position.
*/
post: operations["post_v1_desktop_mouse_move"];
};
"/v1/desktop/mouse/position": {
/**
* Get the current desktop mouse position.
* @description Performs a health-gated mouse position query against the managed desktop.
*/
get: operations["get_v1_desktop_mouse_position"];
};
"/v1/desktop/mouse/scroll": {
/**
* Scroll the desktop mouse wheel.
* @description Performs a health-gated scroll gesture at the requested coordinates and
* returns the resulting mouse position.
*/
post: operations["post_v1_desktop_mouse_scroll"];
};
"/v1/desktop/screenshot": {
/**
* Capture a full desktop screenshot.
* @description Performs a health-gated full-frame screenshot of the managed desktop and
* returns PNG bytes.
*/
get: operations["get_v1_desktop_screenshot"];
};
"/v1/desktop/screenshot/region": {
/**
* Capture a desktop screenshot region.
* @description Performs a health-gated screenshot crop against the managed desktop and
* returns the requested PNG region bytes.
*/
get: operations["get_v1_desktop_screenshot_region"];
};
"/v1/desktop/start": {
/**
* Start the private desktop runtime.
* @description Lazily launches the managed Xvfb/openbox stack, validates display health,
* and returns the resulting desktop status snapshot.
*/
post: operations["post_v1_desktop_start"];
};
"/v1/desktop/status": {
/**
* Get desktop runtime status.
* @description Returns the current desktop runtime state, dependency status, active
* display metadata, and supervised process information.
*/
get: operations["get_v1_desktop_status"];
};
"/v1/desktop/stop": {
/**
* Stop the private desktop runtime.
* @description Terminates the managed openbox/Xvfb/dbus processes owned by the desktop
* runtime and returns the resulting status snapshot.
*/
post: operations["post_v1_desktop_stop"];
};
"/v1/fs/entries": {
get: operations["get_v1_fs_entries"];
};
@ -234,6 +337,119 @@ export interface components {
AgentListResponse: {
agents: components["schemas"]["AgentInfo"][];
};
DesktopActionResponse: {
ok: boolean;
};
DesktopDisplayInfoResponse: {
display: string;
resolution: components["schemas"]["DesktopResolution"];
};
DesktopErrorInfo: {
code: string;
message: string;
};
DesktopKeyboardPressRequest: {
key: string;
};
DesktopKeyboardTypeRequest: {
/** Format: int32 */
delayMs?: number | null;
text: string;
};
/** @enum {string} */
DesktopMouseButton: "left" | "middle" | "right";
DesktopMouseClickRequest: {
button?: components["schemas"]["DesktopMouseButton"] | null;
/** Format: int32 */
clickCount?: number | null;
/** Format: int32 */
x: number;
/** Format: int32 */
y: number;
};
DesktopMouseDragRequest: {
button?: components["schemas"]["DesktopMouseButton"] | null;
/** Format: int32 */
endX: number;
/** Format: int32 */
endY: number;
/** Format: int32 */
startX: number;
/** Format: int32 */
startY: number;
};
DesktopMouseMoveRequest: {
/** Format: int32 */
x: number;
/** Format: int32 */
y: number;
};
DesktopMousePositionResponse: {
/** Format: int32 */
screen?: number | null;
window?: string | null;
/** Format: int32 */
x: number;
/** Format: int32 */
y: number;
};
DesktopMouseScrollRequest: {
/** Format: int32 */
deltaX?: number | null;
/** Format: int32 */
deltaY?: number | null;
/** Format: int32 */
x: number;
/** Format: int32 */
y: number;
};
DesktopProcessInfo: {
logPath?: string | null;
name: string;
/** Format: int32 */
pid?: number | null;
running: boolean;
};
DesktopRegionScreenshotQuery: {
/** Format: int32 */
height: number;
/** Format: int32 */
width: number;
/** Format: int32 */
x: number;
/** Format: int32 */
y: number;
};
DesktopResolution: {
/** Format: int32 */
dpi?: number | null;
/** Format: int32 */
height: number;
/** Format: int32 */
width: number;
};
DesktopScreenshotQuery: Record<string, never>;
DesktopStartRequest: {
/** Format: int32 */
dpi?: number | null;
/** Format: int32 */
height?: number | null;
/** Format: int32 */
width?: number | null;
};
/** @enum {string} */
DesktopState: "inactive" | "install_required" | "starting" | "active" | "stopping" | "failed";
DesktopStatusResponse: {
display?: string | null;
installCommand?: string | null;
lastError?: components["schemas"]["DesktopErrorInfo"] | null;
missingDependencies?: string[];
processes?: components["schemas"]["DesktopProcessInfo"][];
resolution?: components["schemas"]["DesktopResolution"] | null;
runtimeLogPath?: string | null;
startedAt?: string | null;
state: components["schemas"]["DesktopState"];
};
/** @enum {string} */
ErrorType: "invalid_request" | "conflict" | "unsupported_agent" | "agent_not_installed" | "install_failed" | "agent_process_exited" | "token_invalid" | "permission_denied" | "not_acceptable" | "unsupported_media_type" | "not_found" | "session_not_found" | "session_already_exists" | "mode_not_supported" | "stream_error" | "timeout";
FsActionResponse: {
@ -811,6 +1027,441 @@ export interface operations {
};
};
};
/**
* Get desktop display information.
* @description Performs a health-gated display query against the managed desktop and
* returns the current display identifier and resolution.
*/
get_v1_desktop_display_info: {
responses: {
/** @description Desktop display information */
200: {
content: {
"application/json": components["schemas"]["DesktopDisplayInfoResponse"];
};
};
/** @description Desktop runtime is not ready */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime health or display query failed */
503: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
/**
* Press a desktop keyboard shortcut.
* @description Performs a health-gated `xdotool key` operation against the managed
* desktop.
*/
post_v1_desktop_keyboard_press: {
requestBody: {
content: {
"application/json": components["schemas"]["DesktopKeyboardPressRequest"];
};
};
responses: {
/** @description Desktop keyboard action result */
200: {
content: {
"application/json": components["schemas"]["DesktopActionResponse"];
};
};
/** @description Invalid keyboard press request */
400: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime is not ready */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime health or input failed */
503: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
/**
* Type desktop keyboard text.
* @description Performs a health-gated `xdotool type` operation against the managed
* desktop.
*/
post_v1_desktop_keyboard_type: {
requestBody: {
content: {
"application/json": components["schemas"]["DesktopKeyboardTypeRequest"];
};
};
responses: {
/** @description Desktop keyboard action result */
200: {
content: {
"application/json": components["schemas"]["DesktopActionResponse"];
};
};
/** @description Invalid keyboard type request */
400: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime is not ready */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime health or input failed */
503: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
/**
* Click on the desktop.
* @description Performs a health-gated pointer move and click against the managed desktop
* and returns the resulting mouse position.
*/
post_v1_desktop_mouse_click: {
requestBody: {
content: {
"application/json": components["schemas"]["DesktopMouseClickRequest"];
};
};
responses: {
/** @description Desktop mouse position after click */
200: {
content: {
"application/json": components["schemas"]["DesktopMousePositionResponse"];
};
};
/** @description Invalid mouse click request */
400: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime is not ready */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime health or input failed */
503: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
/**
* Drag the desktop mouse.
* @description Performs a health-gated drag gesture against the managed desktop and
* returns the resulting mouse position.
*/
post_v1_desktop_mouse_drag: {
requestBody: {
content: {
"application/json": components["schemas"]["DesktopMouseDragRequest"];
};
};
responses: {
/** @description Desktop mouse position after drag */
200: {
content: {
"application/json": components["schemas"]["DesktopMousePositionResponse"];
};
};
/** @description Invalid mouse drag request */
400: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime is not ready */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime health or input failed */
503: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
/**
* Move the desktop mouse.
* @description Performs a health-gated absolute pointer move on the managed desktop and
* returns the resulting mouse position.
*/
post_v1_desktop_mouse_move: {
requestBody: {
content: {
"application/json": components["schemas"]["DesktopMouseMoveRequest"];
};
};
responses: {
/** @description Desktop mouse position after move */
200: {
content: {
"application/json": components["schemas"]["DesktopMousePositionResponse"];
};
};
/** @description Invalid mouse move request */
400: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime is not ready */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime health or input failed */
503: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
/**
* Get the current desktop mouse position.
* @description Performs a health-gated mouse position query against the managed desktop.
*/
get_v1_desktop_mouse_position: {
responses: {
/** @description Desktop mouse position */
200: {
content: {
"application/json": components["schemas"]["DesktopMousePositionResponse"];
};
};
/** @description Desktop runtime is not ready */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime health or input check failed */
503: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
/**
* Scroll the desktop mouse wheel.
* @description Performs a health-gated scroll gesture at the requested coordinates and
* returns the resulting mouse position.
*/
post_v1_desktop_mouse_scroll: {
requestBody: {
content: {
"application/json": components["schemas"]["DesktopMouseScrollRequest"];
};
};
responses: {
/** @description Desktop mouse position after scroll */
200: {
content: {
"application/json": components["schemas"]["DesktopMousePositionResponse"];
};
};
/** @description Invalid mouse scroll request */
400: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime is not ready */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime health or input failed */
503: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
/**
* Capture a full desktop screenshot.
* @description Performs a health-gated full-frame screenshot of the managed desktop and
* returns PNG bytes.
*/
get_v1_desktop_screenshot: {
responses: {
/** @description Desktop screenshot as PNG bytes */
200: {
content: never;
};
/** @description Desktop runtime is not ready */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime health or screenshot capture failed */
503: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
/**
* Capture a desktop screenshot region.
* @description Performs a health-gated screenshot crop against the managed desktop and
* returns the requested PNG region bytes.
*/
get_v1_desktop_screenshot_region: {
parameters: {
query: {
/** @description Region x coordinate */
x: number;
/** @description Region y coordinate */
y: number;
/** @description Region width */
width: number;
/** @description Region height */
height: number;
};
};
responses: {
/** @description Desktop screenshot region as PNG bytes */
200: {
content: never;
};
/** @description Invalid screenshot region */
400: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime is not ready */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime health or screenshot capture failed */
503: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
/**
* Start the private desktop runtime.
* @description Lazily launches the managed Xvfb/openbox stack, validates display health,
* and returns the resulting desktop status snapshot.
*/
post_v1_desktop_start: {
requestBody: {
content: {
"application/json": components["schemas"]["DesktopStartRequest"];
};
};
responses: {
/** @description Desktop runtime status after start */
200: {
content: {
"application/json": components["schemas"]["DesktopStatusResponse"];
};
};
/** @description Invalid desktop start request */
400: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime is already transitioning */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop API unsupported on this platform */
501: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
/** @description Desktop runtime could not be started */
503: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
/**
* Get desktop runtime status.
* @description Returns the current desktop runtime state, dependency status, active
* display metadata, and supervised process information.
*/
get_v1_desktop_status: {
responses: {
/** @description Desktop runtime status */
200: {
content: {
"application/json": components["schemas"]["DesktopStatusResponse"];
};
};
/** @description Authentication required */
401: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
/**
* Stop the private desktop runtime.
* @description Terminates the managed openbox/Xvfb/dbus processes owned by the desktop
* runtime and returns the resulting status snapshot.
*/
post_v1_desktop_stop: {
responses: {
/** @description Desktop runtime status after stop */
200: {
content: {
"application/json": components["schemas"]["DesktopStatusResponse"];
};
};
/** @description Desktop runtime is already transitioning */
409: {
content: {
"application/json": components["schemas"]["ProblemDetails"];
};
};
};
};
get_v1_fs_entries: {
parameters: {
query?: {

18
sdks/typescript/src/index.ts
View file

@ -45,6 +45,24 @@ export type {
AgentInstallRequest,
AgentInstallResponse,
AgentListResponse,
DesktopActionResponse,
DesktopDisplayInfoResponse,
DesktopErrorInfo,
DesktopKeyboardPressRequest,
DesktopKeyboardTypeRequest,
DesktopMouseButton,
DesktopMouseClickRequest,
DesktopMouseDragRequest,
DesktopMouseMoveRequest,
DesktopMousePositionResponse,
DesktopMouseScrollRequest,
DesktopProcessInfo,
DesktopRegionScreenshotQuery,
DesktopResolution,
DesktopScreenshotQuery,
DesktopStartRequest,
DesktopState,
DesktopStatusResponse,
FsActionResponse,
FsDeleteQuery,
FsEntriesQuery,

21
sdks/typescript/src/types.ts
View file

@ -9,6 +9,27 @@ import type { components, operations } from "./generated/openapi.ts";
export type ProblemDetails = components["schemas"]["ProblemDetails"];
export type HealthResponse = JsonResponse<operations["get_v1_health"], 200>;
export type DesktopState = components["schemas"]["DesktopState"];
export type DesktopResolution = components["schemas"]["DesktopResolution"];
export type DesktopErrorInfo = components["schemas"]["DesktopErrorInfo"];
export type DesktopProcessInfo = components["schemas"]["DesktopProcessInfo"];
export type DesktopStatusResponse = JsonResponse<operations["get_v1_desktop_status"], 200>;
export type DesktopStartRequest = JsonRequestBody<operations["post_v1_desktop_start"]>;
export type DesktopScreenshotQuery =
QueryParams<operations["get_v1_desktop_screenshot"]> extends never
? Record<string, never>
: QueryParams<operations["get_v1_desktop_screenshot"]>;
export type DesktopRegionScreenshotQuery = QueryParams<operations["get_v1_desktop_screenshot_region"]>;
export type DesktopMousePositionResponse = JsonResponse<operations["get_v1_desktop_mouse_position"], 200>;
export type DesktopMouseButton = components["schemas"]["DesktopMouseButton"];
export type DesktopMouseMoveRequest = JsonRequestBody<operations["post_v1_desktop_mouse_move"]>;
export type DesktopMouseClickRequest = JsonRequestBody<operations["post_v1_desktop_mouse_click"]>;
export type DesktopMouseDragRequest = JsonRequestBody<operations["post_v1_desktop_mouse_drag"]>;
export type DesktopMouseScrollRequest = JsonRequestBody<operations["post_v1_desktop_mouse_scroll"]>;
export type DesktopKeyboardTypeRequest = JsonRequestBody<operations["post_v1_desktop_keyboard_type"]>;
export type DesktopKeyboardPressRequest = JsonRequestBody<operations["post_v1_desktop_keyboard_press"]>;
export type DesktopActionResponse = JsonResponse<operations["post_v1_desktop_keyboard_type"], 200>;
export type DesktopDisplayInfoResponse = JsonResponse<operations["get_v1_desktop_display_info"], 200>;
export type AgentListResponse = JsonResponse<operations["get_v1_agents"], 200>;
export type AgentInfo = components["schemas"]["AgentInfo"];
export type AgentQuery = QueryParams<operations["get_v1_agents"]>;