mirror of
https://github.com/harivansh-afk/sandbox-agent.git
synced 2026-04-16 19:04:40 +00:00
Add desktop runtime API and SDK support
This commit is contained in:
Nathan Flurry
parent
3d9476ed0b
commit
641597afe6
27 changed files with 5881 additions and 21 deletions
|
|
@ -25,6 +25,19 @@ import {
|
|||
type AgentInstallRequest,
|
||||
type AgentInstallResponse,
|
||||
type AgentListResponse,
|
||||
type DesktopActionResponse,
|
||||
type DesktopDisplayInfoResponse,
|
||||
type DesktopKeyboardPressRequest,
|
||||
type DesktopKeyboardTypeRequest,
|
||||
type DesktopMouseClickRequest,
|
||||
type DesktopMouseDragRequest,
|
||||
type DesktopMouseMoveRequest,
|
||||
type DesktopMousePositionResponse,
|
||||
type DesktopMouseScrollRequest,
|
||||
type DesktopRegionScreenshotQuery,
|
||||
type DesktopScreenshotQuery,
|
||||
type DesktopStartRequest,
|
||||
type DesktopStatusResponse,
|
||||
type FsActionResponse,
|
||||
type FsDeleteQuery,
|
||||
type FsEntriesQuery,
|
||||
|
|
@ -1294,6 +1307,82 @@ export class SandboxAgent {
|
|||
return this.requestHealth();
|
||||
}
|
||||
|
||||
async startDesktop(request: DesktopStartRequest = {}): Promise<DesktopStatusResponse> {
|
||||
return this.requestJson("POST", `${API_PREFIX}/desktop/start`, {
|
||||
body: request,
|
||||
});
|
||||
}
|
||||
|
||||
async stopDesktop(): Promise<DesktopStatusResponse> {
|
||||
return this.requestJson("POST", `${API_PREFIX}/desktop/stop`);
|
||||
}
|
||||
|
||||
async getDesktopStatus(): Promise<DesktopStatusResponse> {
|
||||
return this.requestJson("GET", `${API_PREFIX}/desktop/status`);
|
||||
}
|
||||
|
||||
async getDesktopDisplayInfo(): Promise<DesktopDisplayInfoResponse> {
|
||||
return this.requestJson("GET", `${API_PREFIX}/desktop/display/info`);
|
||||
}
|
||||
|
||||
async takeDesktopScreenshot(query: DesktopScreenshotQuery = {}): Promise<Uint8Array> {
|
||||
const response = await this.requestRaw("GET", `${API_PREFIX}/desktop/screenshot`, {
|
||||
query,
|
||||
accept: "image/png",
|
||||
});
|
||||
const buffer = await response.arrayBuffer();
|
||||
return new Uint8Array(buffer);
|
||||
}
|
||||
|
||||
async takeDesktopRegionScreenshot(query: DesktopRegionScreenshotQuery): Promise<Uint8Array> {
|
||||
const response = await this.requestRaw("GET", `${API_PREFIX}/desktop/screenshot/region`, {
|
||||
query,
|
||||
accept: "image/png",
|
||||
});
|
||||
const buffer = await response.arrayBuffer();
|
||||
return new Uint8Array(buffer);
|
||||
}
|
||||
|
||||
async getDesktopMousePosition(): Promise<DesktopMousePositionResponse> {
|
||||
return this.requestJson("GET", `${API_PREFIX}/desktop/mouse/position`);
|
||||
}
|
||||
|
||||
async moveDesktopMouse(request: DesktopMouseMoveRequest): Promise<DesktopMousePositionResponse> {
|
||||
return this.requestJson("POST", `${API_PREFIX}/desktop/mouse/move`, {
|
||||
body: request,
|
||||
});
|
||||
}
|
||||
|
||||
async clickDesktop(request: DesktopMouseClickRequest): Promise<DesktopMousePositionResponse> {
|
||||
return this.requestJson("POST", `${API_PREFIX}/desktop/mouse/click`, {
|
||||
body: request,
|
||||
});
|
||||
}
|
||||
|
||||
async dragDesktopMouse(request: DesktopMouseDragRequest): Promise<DesktopMousePositionResponse> {
|
||||
return this.requestJson("POST", `${API_PREFIX}/desktop/mouse/drag`, {
|
||||
body: request,
|
||||
});
|
||||
}
|
||||
|
||||
async scrollDesktop(request: DesktopMouseScrollRequest): Promise<DesktopMousePositionResponse> {
|
||||
return this.requestJson("POST", `${API_PREFIX}/desktop/mouse/scroll`, {
|
||||
body: request,
|
||||
});
|
||||
}
|
||||
|
||||
async typeDesktopText(request: DesktopKeyboardTypeRequest): Promise<DesktopActionResponse> {
|
||||
return this.requestJson("POST", `${API_PREFIX}/desktop/keyboard/type`, {
|
||||
body: request,
|
||||
});
|
||||
}
|
||||
|
||||
async pressDesktopKey(request: DesktopKeyboardPressRequest): Promise<DesktopActionResponse> {
|
||||
return this.requestJson("POST", `${API_PREFIX}/desktop/keyboard/press`, {
|
||||
body: request,
|
||||
});
|
||||
}
|
||||
|
||||
async listAgents(options?: AgentQueryOptions): Promise<AgentListResponse> {
|
||||
return this.requestJson("GET", `${API_PREFIX}/agents`, {
|
||||
query: toAgentQuery(options),
|
||||
|
|
|
|||
|
|
@ -32,6 +32,109 @@ export interface paths {
|
|||
put: operations["put_v1_config_skills"];
|
||||
delete: operations["delete_v1_config_skills"];
|
||||
};
|
||||
"/v1/desktop/display/info": {
|
||||
/**
|
||||
* Get desktop display information.
|
||||
* @description Performs a health-gated display query against the managed desktop and
|
||||
* returns the current display identifier and resolution.
|
||||
*/
|
||||
get: operations["get_v1_desktop_display_info"];
|
||||
};
|
||||
"/v1/desktop/keyboard/press": {
|
||||
/**
|
||||
* Press a desktop keyboard shortcut.
|
||||
* @description Performs a health-gated `xdotool key` operation against the managed
|
||||
* desktop.
|
||||
*/
|
||||
post: operations["post_v1_desktop_keyboard_press"];
|
||||
};
|
||||
"/v1/desktop/keyboard/type": {
|
||||
/**
|
||||
* Type desktop keyboard text.
|
||||
* @description Performs a health-gated `xdotool type` operation against the managed
|
||||
* desktop.
|
||||
*/
|
||||
post: operations["post_v1_desktop_keyboard_type"];
|
||||
};
|
||||
"/v1/desktop/mouse/click": {
|
||||
/**
|
||||
* Click on the desktop.
|
||||
* @description Performs a health-gated pointer move and click against the managed desktop
|
||||
* and returns the resulting mouse position.
|
||||
*/
|
||||
post: operations["post_v1_desktop_mouse_click"];
|
||||
};
|
||||
"/v1/desktop/mouse/drag": {
|
||||
/**
|
||||
* Drag the desktop mouse.
|
||||
* @description Performs a health-gated drag gesture against the managed desktop and
|
||||
* returns the resulting mouse position.
|
||||
*/
|
||||
post: operations["post_v1_desktop_mouse_drag"];
|
||||
};
|
||||
"/v1/desktop/mouse/move": {
|
||||
/**
|
||||
* Move the desktop mouse.
|
||||
* @description Performs a health-gated absolute pointer move on the managed desktop and
|
||||
* returns the resulting mouse position.
|
||||
*/
|
||||
post: operations["post_v1_desktop_mouse_move"];
|
||||
};
|
||||
"/v1/desktop/mouse/position": {
|
||||
/**
|
||||
* Get the current desktop mouse position.
|
||||
* @description Performs a health-gated mouse position query against the managed desktop.
|
||||
*/
|
||||
get: operations["get_v1_desktop_mouse_position"];
|
||||
};
|
||||
"/v1/desktop/mouse/scroll": {
|
||||
/**
|
||||
* Scroll the desktop mouse wheel.
|
||||
* @description Performs a health-gated scroll gesture at the requested coordinates and
|
||||
* returns the resulting mouse position.
|
||||
*/
|
||||
post: operations["post_v1_desktop_mouse_scroll"];
|
||||
};
|
||||
"/v1/desktop/screenshot": {
|
||||
/**
|
||||
* Capture a full desktop screenshot.
|
||||
* @description Performs a health-gated full-frame screenshot of the managed desktop and
|
||||
* returns PNG bytes.
|
||||
*/
|
||||
get: operations["get_v1_desktop_screenshot"];
|
||||
};
|
||||
"/v1/desktop/screenshot/region": {
|
||||
/**
|
||||
* Capture a desktop screenshot region.
|
||||
* @description Performs a health-gated screenshot crop against the managed desktop and
|
||||
* returns the requested PNG region bytes.
|
||||
*/
|
||||
get: operations["get_v1_desktop_screenshot_region"];
|
||||
};
|
||||
"/v1/desktop/start": {
|
||||
/**
|
||||
* Start the private desktop runtime.
|
||||
* @description Lazily launches the managed Xvfb/openbox stack, validates display health,
|
||||
* and returns the resulting desktop status snapshot.
|
||||
*/
|
||||
post: operations["post_v1_desktop_start"];
|
||||
};
|
||||
"/v1/desktop/status": {
|
||||
/**
|
||||
* Get desktop runtime status.
|
||||
* @description Returns the current desktop runtime state, dependency status, active
|
||||
* display metadata, and supervised process information.
|
||||
*/
|
||||
get: operations["get_v1_desktop_status"];
|
||||
};
|
||||
"/v1/desktop/stop": {
|
||||
/**
|
||||
* Stop the private desktop runtime.
|
||||
* @description Terminates the managed openbox/Xvfb/dbus processes owned by the desktop
|
||||
* runtime and returns the resulting status snapshot.
|
||||
*/
|
||||
post: operations["post_v1_desktop_stop"];
|
||||
};
|
||||
"/v1/fs/entries": {
|
||||
get: operations["get_v1_fs_entries"];
|
||||
};
|
||||
|
|
@ -234,6 +337,119 @@ export interface components {
|
|||
AgentListResponse: {
|
||||
agents: components["schemas"]["AgentInfo"][];
|
||||
};
|
||||
DesktopActionResponse: {
|
||||
ok: boolean;
|
||||
};
|
||||
DesktopDisplayInfoResponse: {
|
||||
display: string;
|
||||
resolution: components["schemas"]["DesktopResolution"];
|
||||
};
|
||||
DesktopErrorInfo: {
|
||||
code: string;
|
||||
message: string;
|
||||
};
|
||||
DesktopKeyboardPressRequest: {
|
||||
key: string;
|
||||
};
|
||||
DesktopKeyboardTypeRequest: {
|
||||
/** Format: int32 */
|
||||
delayMs?: number | null;
|
||||
text: string;
|
||||
};
|
||||
/** @enum {string} */
|
||||
DesktopMouseButton: "left" | "middle" | "right";
|
||||
DesktopMouseClickRequest: {
|
||||
button?: components["schemas"]["DesktopMouseButton"] | null;
|
||||
/** Format: int32 */
|
||||
clickCount?: number | null;
|
||||
/** Format: int32 */
|
||||
x: number;
|
||||
/** Format: int32 */
|
||||
y: number;
|
||||
};
|
||||
DesktopMouseDragRequest: {
|
||||
button?: components["schemas"]["DesktopMouseButton"] | null;
|
||||
/** Format: int32 */
|
||||
endX: number;
|
||||
/** Format: int32 */
|
||||
endY: number;
|
||||
/** Format: int32 */
|
||||
startX: number;
|
||||
/** Format: int32 */
|
||||
startY: number;
|
||||
};
|
||||
DesktopMouseMoveRequest: {
|
||||
/** Format: int32 */
|
||||
x: number;
|
||||
/** Format: int32 */
|
||||
y: number;
|
||||
};
|
||||
DesktopMousePositionResponse: {
|
||||
/** Format: int32 */
|
||||
screen?: number | null;
|
||||
window?: string | null;
|
||||
/** Format: int32 */
|
||||
x: number;
|
||||
/** Format: int32 */
|
||||
y: number;
|
||||
};
|
||||
DesktopMouseScrollRequest: {
|
||||
/** Format: int32 */
|
||||
deltaX?: number | null;
|
||||
/** Format: int32 */
|
||||
deltaY?: number | null;
|
||||
/** Format: int32 */
|
||||
x: number;
|
||||
/** Format: int32 */
|
||||
y: number;
|
||||
};
|
||||
DesktopProcessInfo: {
|
||||
logPath?: string | null;
|
||||
name: string;
|
||||
/** Format: int32 */
|
||||
pid?: number | null;
|
||||
running: boolean;
|
||||
};
|
||||
DesktopRegionScreenshotQuery: {
|
||||
/** Format: int32 */
|
||||
height: number;
|
||||
/** Format: int32 */
|
||||
width: number;
|
||||
/** Format: int32 */
|
||||
x: number;
|
||||
/** Format: int32 */
|
||||
y: number;
|
||||
};
|
||||
DesktopResolution: {
|
||||
/** Format: int32 */
|
||||
dpi?: number | null;
|
||||
/** Format: int32 */
|
||||
height: number;
|
||||
/** Format: int32 */
|
||||
width: number;
|
||||
};
|
||||
DesktopScreenshotQuery: Record<string, never>;
|
||||
DesktopStartRequest: {
|
||||
/** Format: int32 */
|
||||
dpi?: number | null;
|
||||
/** Format: int32 */
|
||||
height?: number | null;
|
||||
/** Format: int32 */
|
||||
width?: number | null;
|
||||
};
|
||||
/** @enum {string} */
|
||||
DesktopState: "inactive" | "install_required" | "starting" | "active" | "stopping" | "failed";
|
||||
DesktopStatusResponse: {
|
||||
display?: string | null;
|
||||
installCommand?: string | null;
|
||||
lastError?: components["schemas"]["DesktopErrorInfo"] | null;
|
||||
missingDependencies?: string[];
|
||||
processes?: components["schemas"]["DesktopProcessInfo"][];
|
||||
resolution?: components["schemas"]["DesktopResolution"] | null;
|
||||
runtimeLogPath?: string | null;
|
||||
startedAt?: string | null;
|
||||
state: components["schemas"]["DesktopState"];
|
||||
};
|
||||
/** @enum {string} */
|
||||
ErrorType: "invalid_request" | "conflict" | "unsupported_agent" | "agent_not_installed" | "install_failed" | "agent_process_exited" | "token_invalid" | "permission_denied" | "not_acceptable" | "unsupported_media_type" | "not_found" | "session_not_found" | "session_already_exists" | "mode_not_supported" | "stream_error" | "timeout";
|
||||
FsActionResponse: {
|
||||
|
|
@ -811,6 +1027,441 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
/**
|
||||
* Get desktop display information.
|
||||
* @description Performs a health-gated display query against the managed desktop and
|
||||
* returns the current display identifier and resolution.
|
||||
*/
|
||||
get_v1_desktop_display_info: {
|
||||
responses: {
|
||||
/** @description Desktop display information */
|
||||
200: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["DesktopDisplayInfoResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime is not ready */
|
||||
409: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime health or display query failed */
|
||||
503: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
/**
|
||||
* Press a desktop keyboard shortcut.
|
||||
* @description Performs a health-gated `xdotool key` operation against the managed
|
||||
* desktop.
|
||||
*/
|
||||
post_v1_desktop_keyboard_press: {
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["DesktopKeyboardPressRequest"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Desktop keyboard action result */
|
||||
200: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["DesktopActionResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Invalid keyboard press request */
|
||||
400: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime is not ready */
|
||||
409: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime health or input failed */
|
||||
503: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
/**
|
||||
* Type desktop keyboard text.
|
||||
* @description Performs a health-gated `xdotool type` operation against the managed
|
||||
* desktop.
|
||||
*/
|
||||
post_v1_desktop_keyboard_type: {
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["DesktopKeyboardTypeRequest"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Desktop keyboard action result */
|
||||
200: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["DesktopActionResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Invalid keyboard type request */
|
||||
400: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime is not ready */
|
||||
409: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime health or input failed */
|
||||
503: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
/**
|
||||
* Click on the desktop.
|
||||
* @description Performs a health-gated pointer move and click against the managed desktop
|
||||
* and returns the resulting mouse position.
|
||||
*/
|
||||
post_v1_desktop_mouse_click: {
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["DesktopMouseClickRequest"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Desktop mouse position after click */
|
||||
200: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["DesktopMousePositionResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Invalid mouse click request */
|
||||
400: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime is not ready */
|
||||
409: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime health or input failed */
|
||||
503: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
/**
|
||||
* Drag the desktop mouse.
|
||||
* @description Performs a health-gated drag gesture against the managed desktop and
|
||||
* returns the resulting mouse position.
|
||||
*/
|
||||
post_v1_desktop_mouse_drag: {
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["DesktopMouseDragRequest"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Desktop mouse position after drag */
|
||||
200: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["DesktopMousePositionResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Invalid mouse drag request */
|
||||
400: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime is not ready */
|
||||
409: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime health or input failed */
|
||||
503: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
/**
|
||||
* Move the desktop mouse.
|
||||
* @description Performs a health-gated absolute pointer move on the managed desktop and
|
||||
* returns the resulting mouse position.
|
||||
*/
|
||||
post_v1_desktop_mouse_move: {
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["DesktopMouseMoveRequest"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Desktop mouse position after move */
|
||||
200: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["DesktopMousePositionResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Invalid mouse move request */
|
||||
400: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime is not ready */
|
||||
409: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime health or input failed */
|
||||
503: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
/**
|
||||
* Get the current desktop mouse position.
|
||||
* @description Performs a health-gated mouse position query against the managed desktop.
|
||||
*/
|
||||
get_v1_desktop_mouse_position: {
|
||||
responses: {
|
||||
/** @description Desktop mouse position */
|
||||
200: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["DesktopMousePositionResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime is not ready */
|
||||
409: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime health or input check failed */
|
||||
503: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
/**
|
||||
* Scroll the desktop mouse wheel.
|
||||
* @description Performs a health-gated scroll gesture at the requested coordinates and
|
||||
* returns the resulting mouse position.
|
||||
*/
|
||||
post_v1_desktop_mouse_scroll: {
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["DesktopMouseScrollRequest"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Desktop mouse position after scroll */
|
||||
200: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["DesktopMousePositionResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Invalid mouse scroll request */
|
||||
400: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime is not ready */
|
||||
409: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime health or input failed */
|
||||
503: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
/**
|
||||
* Capture a full desktop screenshot.
|
||||
* @description Performs a health-gated full-frame screenshot of the managed desktop and
|
||||
* returns PNG bytes.
|
||||
*/
|
||||
get_v1_desktop_screenshot: {
|
||||
responses: {
|
||||
/** @description Desktop screenshot as PNG bytes */
|
||||
200: {
|
||||
content: never;
|
||||
};
|
||||
/** @description Desktop runtime is not ready */
|
||||
409: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime health or screenshot capture failed */
|
||||
503: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
/**
|
||||
* Capture a desktop screenshot region.
|
||||
* @description Performs a health-gated screenshot crop against the managed desktop and
|
||||
* returns the requested PNG region bytes.
|
||||
*/
|
||||
get_v1_desktop_screenshot_region: {
|
||||
parameters: {
|
||||
query: {
|
||||
/** @description Region x coordinate */
|
||||
x: number;
|
||||
/** @description Region y coordinate */
|
||||
y: number;
|
||||
/** @description Region width */
|
||||
width: number;
|
||||
/** @description Region height */
|
||||
height: number;
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Desktop screenshot region as PNG bytes */
|
||||
200: {
|
||||
content: never;
|
||||
};
|
||||
/** @description Invalid screenshot region */
|
||||
400: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime is not ready */
|
||||
409: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime health or screenshot capture failed */
|
||||
503: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
/**
|
||||
* Start the private desktop runtime.
|
||||
* @description Lazily launches the managed Xvfb/openbox stack, validates display health,
|
||||
* and returns the resulting desktop status snapshot.
|
||||
*/
|
||||
post_v1_desktop_start: {
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["DesktopStartRequest"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Desktop runtime status after start */
|
||||
200: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["DesktopStatusResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Invalid desktop start request */
|
||||
400: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime is already transitioning */
|
||||
409: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop API unsupported on this platform */
|
||||
501: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime could not be started */
|
||||
503: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
/**
|
||||
* Get desktop runtime status.
|
||||
* @description Returns the current desktop runtime state, dependency status, active
|
||||
* display metadata, and supervised process information.
|
||||
*/
|
||||
get_v1_desktop_status: {
|
||||
responses: {
|
||||
/** @description Desktop runtime status */
|
||||
200: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["DesktopStatusResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Authentication required */
|
||||
401: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
/**
|
||||
* Stop the private desktop runtime.
|
||||
* @description Terminates the managed openbox/Xvfb/dbus processes owned by the desktop
|
||||
* runtime and returns the resulting status snapshot.
|
||||
*/
|
||||
post_v1_desktop_stop: {
|
||||
responses: {
|
||||
/** @description Desktop runtime status after stop */
|
||||
200: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["DesktopStatusResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Desktop runtime is already transitioning */
|
||||
409: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["ProblemDetails"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
get_v1_fs_entries: {
|
||||
parameters: {
|
||||
query?: {
|
||||
|
|
|
|||
|
|
@ -45,6 +45,24 @@ export type {
|
|||
AgentInstallRequest,
|
||||
AgentInstallResponse,
|
||||
AgentListResponse,
|
||||
DesktopActionResponse,
|
||||
DesktopDisplayInfoResponse,
|
||||
DesktopErrorInfo,
|
||||
DesktopKeyboardPressRequest,
|
||||
DesktopKeyboardTypeRequest,
|
||||
DesktopMouseButton,
|
||||
DesktopMouseClickRequest,
|
||||
DesktopMouseDragRequest,
|
||||
DesktopMouseMoveRequest,
|
||||
DesktopMousePositionResponse,
|
||||
DesktopMouseScrollRequest,
|
||||
DesktopProcessInfo,
|
||||
DesktopRegionScreenshotQuery,
|
||||
DesktopResolution,
|
||||
DesktopScreenshotQuery,
|
||||
DesktopStartRequest,
|
||||
DesktopState,
|
||||
DesktopStatusResponse,
|
||||
FsActionResponse,
|
||||
FsDeleteQuery,
|
||||
FsEntriesQuery,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,27 @@ import type { components, operations } from "./generated/openapi.ts";
|
|||
export type ProblemDetails = components["schemas"]["ProblemDetails"];
|
||||
|
||||
export type HealthResponse = JsonResponse<operations["get_v1_health"], 200>;
|
||||
export type DesktopState = components["schemas"]["DesktopState"];
|
||||
export type DesktopResolution = components["schemas"]["DesktopResolution"];
|
||||
export type DesktopErrorInfo = components["schemas"]["DesktopErrorInfo"];
|
||||
export type DesktopProcessInfo = components["schemas"]["DesktopProcessInfo"];
|
||||
export type DesktopStatusResponse = JsonResponse<operations["get_v1_desktop_status"], 200>;
|
||||
export type DesktopStartRequest = JsonRequestBody<operations["post_v1_desktop_start"]>;
|
||||
export type DesktopScreenshotQuery =
|
||||
QueryParams<operations["get_v1_desktop_screenshot"]> extends never
|
||||
? Record<string, never>
|
||||
: QueryParams<operations["get_v1_desktop_screenshot"]>;
|
||||
export type DesktopRegionScreenshotQuery = QueryParams<operations["get_v1_desktop_screenshot_region"]>;
|
||||
export type DesktopMousePositionResponse = JsonResponse<operations["get_v1_desktop_mouse_position"], 200>;
|
||||
export type DesktopMouseButton = components["schemas"]["DesktopMouseButton"];
|
||||
export type DesktopMouseMoveRequest = JsonRequestBody<operations["post_v1_desktop_mouse_move"]>;
|
||||
export type DesktopMouseClickRequest = JsonRequestBody<operations["post_v1_desktop_mouse_click"]>;
|
||||
export type DesktopMouseDragRequest = JsonRequestBody<operations["post_v1_desktop_mouse_drag"]>;
|
||||
export type DesktopMouseScrollRequest = JsonRequestBody<operations["post_v1_desktop_mouse_scroll"]>;
|
||||
export type DesktopKeyboardTypeRequest = JsonRequestBody<operations["post_v1_desktop_keyboard_type"]>;
|
||||
export type DesktopKeyboardPressRequest = JsonRequestBody<operations["post_v1_desktop_keyboard_press"]>;
|
||||
export type DesktopActionResponse = JsonResponse<operations["post_v1_desktop_keyboard_type"], 200>;
|
||||
export type DesktopDisplayInfoResponse = JsonResponse<operations["get_v1_desktop_display_info"], 200>;
|
||||
export type AgentListResponse = JsonResponse<operations["get_v1_agents"], 200>;
|
||||
export type AgentInfo = components["schemas"]["AgentInfo"];
|
||||
export type AgentQuery = QueryParams<operations["get_v1_agents"]>;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { describe, it, expect, beforeAll, afterAll } from "vitest";
|
||||
import { existsSync } from "node:fs";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
|
@ -150,21 +150,166 @@ function nodeCommand(source: string): { command: string; args: string[] } {
|
|||
};
|
||||
}
|
||||
|
||||
function writeExecutable(path: string, source: string): void {
|
||||
writeFileSync(path, source, "utf8");
|
||||
chmodSync(path, 0o755);
|
||||
}
|
||||
|
||||
function prepareFakeDesktopEnv(root: string): Record<string, string> {
|
||||
const binDir = join(root, "bin");
|
||||
const xdgStateHome = join(root, "xdg-state");
|
||||
const fakeStateDir = join(root, "fake-state");
|
||||
mkdirSync(binDir, { recursive: true });
|
||||
mkdirSync(xdgStateHome, { recursive: true });
|
||||
mkdirSync(fakeStateDir, { recursive: true });
|
||||
|
||||
writeExecutable(
|
||||
join(binDir, "Xvfb"),
|
||||
`#!/usr/bin/env sh
|
||||
set -eu
|
||||
display="\${1:-:191}"
|
||||
number="\${display#:}"
|
||||
socket="/tmp/.X11-unix/X\${number}"
|
||||
mkdir -p /tmp/.X11-unix
|
||||
touch "$socket"
|
||||
cleanup() {
|
||||
rm -f "$socket"
|
||||
exit 0
|
||||
}
|
||||
trap cleanup INT TERM EXIT
|
||||
while :; do
|
||||
sleep 1
|
||||
done
|
||||
`,
|
||||
);
|
||||
|
||||
writeExecutable(
|
||||
join(binDir, "openbox"),
|
||||
`#!/usr/bin/env sh
|
||||
set -eu
|
||||
trap 'exit 0' INT TERM
|
||||
while :; do
|
||||
sleep 1
|
||||
done
|
||||
`,
|
||||
);
|
||||
|
||||
writeExecutable(
|
||||
join(binDir, "dbus-launch"),
|
||||
`#!/usr/bin/env sh
|
||||
set -eu
|
||||
echo "DBUS_SESSION_BUS_ADDRESS=unix:path=/tmp/sandbox-agent-test-bus"
|
||||
echo "DBUS_SESSION_BUS_PID=$$"
|
||||
`,
|
||||
);
|
||||
|
||||
writeExecutable(
|
||||
join(binDir, "xrandr"),
|
||||
`#!/usr/bin/env sh
|
||||
set -eu
|
||||
cat <<'EOF'
|
||||
Screen 0: minimum 1 x 1, current 1440 x 900, maximum 32767 x 32767
|
||||
EOF
|
||||
`,
|
||||
);
|
||||
|
||||
writeExecutable(
|
||||
join(binDir, "import"),
|
||||
`#!/usr/bin/env sh
|
||||
set -eu
|
||||
printf '\\211PNG\\r\\n\\032\\n\\000\\000\\000\\rIHDR\\000\\000\\000\\001\\000\\000\\000\\001\\010\\006\\000\\000\\000\\037\\025\\304\\211\\000\\000\\000\\013IDATx\\234c\\000\\001\\000\\000\\005\\000\\001\\r\\n-\\264\\000\\000\\000\\000IEND\\256B\`\\202'
|
||||
`,
|
||||
);
|
||||
|
||||
writeExecutable(
|
||||
join(binDir, "xdotool"),
|
||||
`#!/usr/bin/env sh
|
||||
set -eu
|
||||
state_dir="\${SANDBOX_AGENT_DESKTOP_FAKE_STATE_DIR:?missing fake state dir}"
|
||||
state_file="\${state_dir}/mouse"
|
||||
mkdir -p "$state_dir"
|
||||
if [ ! -f "$state_file" ]; then
|
||||
printf '0 0\\n' > "$state_file"
|
||||
fi
|
||||
|
||||
read_state() {
|
||||
read -r x y < "$state_file"
|
||||
}
|
||||
|
||||
write_state() {
|
||||
printf '%s %s\\n' "$1" "$2" > "$state_file"
|
||||
}
|
||||
|
||||
command="\${1:-}"
|
||||
case "$command" in
|
||||
getmouselocation)
|
||||
read_state
|
||||
printf 'X=%s\\nY=%s\\nSCREEN=0\\nWINDOW=0\\n' "$x" "$y"
|
||||
;;
|
||||
mousemove)
|
||||
shift
|
||||
x="\${1:-0}"
|
||||
y="\${2:-0}"
|
||||
shift 2 || true
|
||||
while [ "$#" -gt 0 ]; do
|
||||
token="$1"
|
||||
shift
|
||||
case "$token" in
|
||||
mousemove)
|
||||
x="\${1:-0}"
|
||||
y="\${2:-0}"
|
||||
shift 2 || true
|
||||
;;
|
||||
mousedown|mouseup)
|
||||
shift 1 || true
|
||||
;;
|
||||
click)
|
||||
if [ "\${1:-}" = "--repeat" ]; then
|
||||
shift 2 || true
|
||||
fi
|
||||
shift 1 || true
|
||||
;;
|
||||
esac
|
||||
done
|
||||
write_state "$x" "$y"
|
||||
;;
|
||||
type|key)
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
`,
|
||||
);
|
||||
|
||||
return {
|
||||
SANDBOX_AGENT_DESKTOP_TEST_ASSUME_LINUX: "1",
|
||||
SANDBOX_AGENT_DESKTOP_DISPLAY_NUM: "191",
|
||||
SANDBOX_AGENT_DESKTOP_FAKE_STATE_DIR: fakeStateDir,
|
||||
XDG_STATE_HOME: xdgStateHome,
|
||||
PATH: `${binDir}:${process.env.PATH ?? ""}`,
|
||||
};
|
||||
}
|
||||
|
||||
describe("Integration: TypeScript SDK flat session API", () => {
|
||||
let handle: SandboxAgentSpawnHandle;
|
||||
let baseUrl: string;
|
||||
let token: string;
|
||||
let dataHome: string;
|
||||
let desktopHome: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
dataHome = mkdtempSync(join(tmpdir(), "sdk-integration-"));
|
||||
desktopHome = mkdtempSync(join(tmpdir(), "sdk-desktop-"));
|
||||
const agentEnv = prepareMockAgentDataHome(dataHome);
|
||||
const desktopEnv = prepareFakeDesktopEnv(desktopHome);
|
||||
|
||||
handle = await spawnSandboxAgent({
|
||||
enabled: true,
|
||||
log: "silent",
|
||||
timeoutMs: 30000,
|
||||
env: agentEnv,
|
||||
env: { ...agentEnv, ...desktopEnv },
|
||||
});
|
||||
baseUrl = handle.baseUrl;
|
||||
token = handle.token;
|
||||
|
|
@ -173,6 +318,7 @@ describe("Integration: TypeScript SDK flat session API", () => {
|
|||
afterAll(async () => {
|
||||
await handle.dispose();
|
||||
rmSync(dataHome, { recursive: true, force: true });
|
||||
rmSync(desktopHome, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("detects Node.js runtime", () => {
|
||||
|
|
@ -882,4 +1028,91 @@ describe("Integration: TypeScript SDK flat session API", () => {
|
|||
await sdk.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it("covers desktop status, screenshot, display, mouse, and keyboard helpers", async () => {
|
||||
const sdk = await SandboxAgent.connect({
|
||||
baseUrl,
|
||||
token,
|
||||
});
|
||||
|
||||
try {
|
||||
const initialStatus = await sdk.getDesktopStatus();
|
||||
expect(initialStatus.state).toBe("inactive");
|
||||
|
||||
const started = await sdk.startDesktop({
|
||||
width: 1440,
|
||||
height: 900,
|
||||
dpi: 96,
|
||||
});
|
||||
expect(started.state).toBe("active");
|
||||
expect(started.display?.startsWith(":")).toBe(true);
|
||||
expect(started.missingDependencies).toEqual([]);
|
||||
|
||||
const displayInfo = await sdk.getDesktopDisplayInfo();
|
||||
expect(displayInfo.display).toBe(started.display);
|
||||
expect(displayInfo.resolution.width).toBe(1440);
|
||||
expect(displayInfo.resolution.height).toBe(900);
|
||||
|
||||
const screenshot = await sdk.takeDesktopScreenshot();
|
||||
expect(Buffer.from(screenshot.subarray(0, 8)).equals(Buffer.from("\x89PNG\r\n\x1a\n", "binary"))).toBe(true);
|
||||
|
||||
const region = await sdk.takeDesktopRegionScreenshot({
|
||||
x: 10,
|
||||
y: 20,
|
||||
width: 40,
|
||||
height: 50,
|
||||
});
|
||||
expect(Buffer.from(region.subarray(0, 8)).equals(Buffer.from("\x89PNG\r\n\x1a\n", "binary"))).toBe(true);
|
||||
|
||||
const moved = await sdk.moveDesktopMouse({ x: 40, y: 50 });
|
||||
expect(moved.x).toBe(40);
|
||||
expect(moved.y).toBe(50);
|
||||
|
||||
const dragged = await sdk.dragDesktopMouse({
|
||||
startX: 40,
|
||||
startY: 50,
|
||||
endX: 80,
|
||||
endY: 90,
|
||||
button: "left",
|
||||
});
|
||||
expect(dragged.x).toBe(80);
|
||||
expect(dragged.y).toBe(90);
|
||||
|
||||
const clicked = await sdk.clickDesktop({
|
||||
x: 80,
|
||||
y: 90,
|
||||
button: "left",
|
||||
clickCount: 1,
|
||||
});
|
||||
expect(clicked.x).toBe(80);
|
||||
expect(clicked.y).toBe(90);
|
||||
|
||||
const scrolled = await sdk.scrollDesktop({
|
||||
x: 80,
|
||||
y: 90,
|
||||
deltaY: -2,
|
||||
});
|
||||
expect(scrolled.x).toBe(80);
|
||||
expect(scrolled.y).toBe(90);
|
||||
|
||||
const position = await sdk.getDesktopMousePosition();
|
||||
expect(position.x).toBe(80);
|
||||
expect(position.y).toBe(90);
|
||||
|
||||
const typed = await sdk.typeDesktopText({
|
||||
text: "hello desktop",
|
||||
delayMs: 5,
|
||||
});
|
||||
expect(typed.ok).toBe(true);
|
||||
|
||||
const pressed = await sdk.pressDesktopKey({ key: "ctrl+l" });
|
||||
expect(pressed.ok).toBe(true);
|
||||
|
||||
const stopped = await sdk.stopDesktop();
|
||||
expect(stopped.state).toBe("inactive");
|
||||
} finally {
|
||||
await sdk.stopDesktop().catch(() => {});
|
||||
await sdk.dispose();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue