From c74d8c917996939a919c802c3c7e378f83783c12 Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Sat, 7 Mar 2026 23:32:49 -0800 Subject: [PATCH] Add desktop runtime API and SDK support --- docs/cli.mdx | 30 + docs/deploy/docker.mdx | 21 + docs/inspector.mdx | 14 + docs/openapi.json | 1100 ++++++++++++++ docs/quickstart.mdx | 10 + docs/sdk-overview.mdx | 38 + frontend/packages/inspector/index.html | 93 ++ frontend/packages/inspector/package.json | 1 + .../src/components/debug/DebugPanel.tsx | 17 +- .../src/components/debug/DesktopTab.test.tsx | 150 ++ .../src/components/debug/DesktopTab.tsx | 373 +++++ pnpm-lock.yaml | 424 +++++- research/acp/friction.md | 10 + sdks/typescript/src/client.ts | 89 ++ sdks/typescript/src/generated/openapi.ts | 651 ++++++++ sdks/typescript/src/index.ts | 18 + sdks/typescript/src/types.ts | 21 + sdks/typescript/tests/integration.test.ts | 237 ++- server/packages/sandbox-agent/src/cli.rs | 47 + .../sandbox-agent/src/desktop_errors.rs | 158 ++ .../sandbox-agent/src/desktop_install.rs | 282 ++++ .../sandbox-agent/src/desktop_runtime.rs | 1313 +++++++++++++++++ .../sandbox-agent/src/desktop_types.rs | 173 +++ server/packages/sandbox-agent/src/lib.rs | 4 + server/packages/sandbox-agent/src/router.rs | 363 +++++ server/packages/sandbox-agent/tests/v1_api.rs | 158 ++ .../sandbox-agent/tests/v1_api/desktop.rs | 222 +++ 27 files changed, 5938 insertions(+), 79 deletions(-) create mode 100644 frontend/packages/inspector/src/components/debug/DesktopTab.test.tsx create mode 100644 frontend/packages/inspector/src/components/debug/DesktopTab.tsx create mode 100644 server/packages/sandbox-agent/src/desktop_errors.rs create mode 100644 server/packages/sandbox-agent/src/desktop_install.rs create mode 100644 server/packages/sandbox-agent/src/desktop_runtime.rs create mode 100644 server/packages/sandbox-agent/src/desktop_types.rs create mode 100644 server/packages/sandbox-agent/tests/v1_api/desktop.rs diff --git a/docs/cli.mdx b/docs/cli.mdx index a3cd839..5ec4c24 100644 --- a/docs/cli.mdx +++ b/docs/cli.mdx @@ -37,6 +37,36 @@ Notes: - Set `SANDBOX_AGENT_LOG_STDOUT=1` to force stdout/stderr logging. - Use `SANDBOX_AGENT_LOG_DIR` to override log directory. +## install + +Install first-party runtime dependencies. + +### install desktop + +Install the Linux desktop runtime packages required by `/v1/desktop/*`. + +```bash +sandbox-agent install desktop [OPTIONS] +``` + +| Option | Description | +|--------|-------------| +| `--yes` | Skip the confirmation prompt | +| `--print-only` | Print the package-manager command without executing it | +| `--package-manager ` | Override package-manager detection | +| `--no-fonts` | Skip the default DejaVu font package | + +```bash +sandbox-agent install desktop --yes +sandbox-agent install desktop --print-only +``` + +Notes: + +- Supported on Linux only. +- The command detects `apt`, `dnf`, or `apk`. +- If the host is not already running as root, the command requires `sudo`. + ## install-agent Install or reinstall a single agent, or every supported agent with `--all`. diff --git a/docs/deploy/docker.mdx b/docs/deploy/docker.mdx index 030ddc9..4eb1bfc 100644 --- a/docs/deploy/docker.mdx +++ b/docs/deploy/docker.mdx @@ -21,6 +21,27 @@ docker run --rm -p 3000:3000 \ The `0.3.1-full` tag pins the exact version. The moving `full` tag is also published for contributors who want the latest full image. +If you also want the desktop API inside the container, install desktop dependencies before starting the server: + +```bash +docker run --rm -p 3000:3000 \ + -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" \ + -e OPENAI_API_KEY="$OPENAI_API_KEY" \ + node:22-bookworm-slim sh -c "\ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y curl ca-certificates bash libstdc++6 && \ + rm -rf /var/lib/apt/lists/* && \ + curl -fsSL https://releases.rivet.dev/sandbox-agent/0.3.x/install.sh | sh && \ + sandbox-agent install desktop --yes && \ + sandbox-agent server --no-token --host 0.0.0.0 --port 3000" +``` + +In a Dockerfile: + +```dockerfile +RUN sandbox-agent install desktop --yes +``` + ## TypeScript with dockerode ```typescript diff --git a/docs/inspector.mdx b/docs/inspector.mdx index cc5f3d0..1412c21 100644 --- a/docs/inspector.mdx +++ b/docs/inspector.mdx @@ -35,6 +35,7 @@ console.log(url); - Prompt testing - Request/response debugging - Interactive permission prompts (approve, always-allow, or reject tool-use requests) +- Desktop panel for status, remediation, start/stop, and screenshot refresh - Process management (create, stop, kill, delete, view logs) - Interactive PTY terminal for tty processes - One-shot command execution @@ -50,3 +51,16 @@ console.log(url); The Inspector includes an embedded Ghostty-based terminal for interactive tty processes. The UI uses the SDK's high-level `connectProcessTerminal(...)` wrapper via the shared `@sandbox-agent/react` `ProcessTerminal` component. + +## Desktop panel + +The `Desktop` panel shows the current desktop runtime state, missing dependencies, +the suggested install command, last error details, process/log paths, and the +latest captured screenshot. + +Use it to: + +- Check whether desktop dependencies are installed +- Start or stop the managed desktop runtime +- Refresh desktop status +- Capture a fresh screenshot on demand diff --git a/docs/openapi.json b/docs/openapi.json index f2bd640..c8d23b5 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -628,6 +628,722 @@ } } }, + "/v1/desktop/display/info": { + "get": { + "tags": [ + "v1" + ], + "summary": "Get desktop display information.", + "description": "Performs a health-gated display query against the managed desktop and\nreturns the current display identifier and resolution.", + "operationId": "get_v1_desktop_display_info", + "responses": { + "200": { + "description": "Desktop display information", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopDisplayInfoResponse" + } + } + } + }, + "409": { + "description": "Desktop runtime is not ready", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "503": { + "description": "Desktop runtime health or display query failed", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/desktop/keyboard/press": { + "post": { + "tags": [ + "v1" + ], + "summary": "Press a desktop keyboard shortcut.", + "description": "Performs a health-gated `xdotool key` operation against the managed\ndesktop.", + "operationId": "post_v1_desktop_keyboard_press", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopKeyboardPressRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Desktop keyboard action result", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopActionResponse" + } + } + } + }, + "400": { + "description": "Invalid keyboard press request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "409": { + "description": "Desktop runtime is not ready", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "503": { + "description": "Desktop runtime health or input failed", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/desktop/keyboard/type": { + "post": { + "tags": [ + "v1" + ], + "summary": "Type desktop keyboard text.", + "description": "Performs a health-gated `xdotool type` operation against the managed\ndesktop.", + "operationId": "post_v1_desktop_keyboard_type", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopKeyboardTypeRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Desktop keyboard action result", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopActionResponse" + } + } + } + }, + "400": { + "description": "Invalid keyboard type request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "409": { + "description": "Desktop runtime is not ready", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "503": { + "description": "Desktop runtime health or input failed", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/desktop/mouse/click": { + "post": { + "tags": [ + "v1" + ], + "summary": "Click on the desktop.", + "description": "Performs a health-gated pointer move and click against the managed desktop\nand returns the resulting mouse position.", + "operationId": "post_v1_desktop_mouse_click", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopMouseClickRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Desktop mouse position after click", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopMousePositionResponse" + } + } + } + }, + "400": { + "description": "Invalid mouse click request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "409": { + "description": "Desktop runtime is not ready", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "503": { + "description": "Desktop runtime health or input failed", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/desktop/mouse/drag": { + "post": { + "tags": [ + "v1" + ], + "summary": "Drag the desktop mouse.", + "description": "Performs a health-gated drag gesture against the managed desktop and\nreturns the resulting mouse position.", + "operationId": "post_v1_desktop_mouse_drag", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopMouseDragRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Desktop mouse position after drag", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopMousePositionResponse" + } + } + } + }, + "400": { + "description": "Invalid mouse drag request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "409": { + "description": "Desktop runtime is not ready", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "503": { + "description": "Desktop runtime health or input failed", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/desktop/mouse/move": { + "post": { + "tags": [ + "v1" + ], + "summary": "Move the desktop mouse.", + "description": "Performs a health-gated absolute pointer move on the managed desktop and\nreturns the resulting mouse position.", + "operationId": "post_v1_desktop_mouse_move", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopMouseMoveRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Desktop mouse position after move", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopMousePositionResponse" + } + } + } + }, + "400": { + "description": "Invalid mouse move request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "409": { + "description": "Desktop runtime is not ready", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "503": { + "description": "Desktop runtime health or input failed", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/desktop/mouse/position": { + "get": { + "tags": [ + "v1" + ], + "summary": "Get the current desktop mouse position.", + "description": "Performs a health-gated mouse position query against the managed desktop.", + "operationId": "get_v1_desktop_mouse_position", + "responses": { + "200": { + "description": "Desktop mouse position", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopMousePositionResponse" + } + } + } + }, + "409": { + "description": "Desktop runtime is not ready", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "503": { + "description": "Desktop runtime health or input check failed", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/desktop/mouse/scroll": { + "post": { + "tags": [ + "v1" + ], + "summary": "Scroll the desktop mouse wheel.", + "description": "Performs a health-gated scroll gesture at the requested coordinates and\nreturns the resulting mouse position.", + "operationId": "post_v1_desktop_mouse_scroll", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopMouseScrollRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Desktop mouse position after scroll", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopMousePositionResponse" + } + } + } + }, + "400": { + "description": "Invalid mouse scroll request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "409": { + "description": "Desktop runtime is not ready", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "503": { + "description": "Desktop runtime health or input failed", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/desktop/screenshot": { + "get": { + "tags": [ + "v1" + ], + "summary": "Capture a full desktop screenshot.", + "description": "Performs a health-gated full-frame screenshot of the managed desktop and\nreturns PNG bytes.", + "operationId": "get_v1_desktop_screenshot", + "responses": { + "200": { + "description": "Desktop screenshot as PNG bytes" + }, + "409": { + "description": "Desktop runtime is not ready", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "503": { + "description": "Desktop runtime health or screenshot capture failed", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/desktop/screenshot/region": { + "get": { + "tags": [ + "v1" + ], + "summary": "Capture a desktop screenshot region.", + "description": "Performs a health-gated screenshot crop against the managed desktop and\nreturns the requested PNG region bytes.", + "operationId": "get_v1_desktop_screenshot_region", + "parameters": [ + { + "name": "x", + "in": "query", + "description": "Region x coordinate", + "required": true, + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "y", + "in": "query", + "description": "Region y coordinate", + "required": true, + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "width", + "in": "query", + "description": "Region width", + "required": true, + "schema": { + "type": "integer", + "format": "int32", + "minimum": 0 + } + }, + { + "name": "height", + "in": "query", + "description": "Region height", + "required": true, + "schema": { + "type": "integer", + "format": "int32", + "minimum": 0 + } + } + ], + "responses": { + "200": { + "description": "Desktop screenshot region as PNG bytes" + }, + "400": { + "description": "Invalid screenshot region", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "409": { + "description": "Desktop runtime is not ready", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "503": { + "description": "Desktop runtime health or screenshot capture failed", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/desktop/start": { + "post": { + "tags": [ + "v1" + ], + "summary": "Start the private desktop runtime.", + "description": "Lazily launches the managed Xvfb/openbox stack, validates display health,\nand returns the resulting desktop status snapshot.", + "operationId": "post_v1_desktop_start", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopStartRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Desktop runtime status after start", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopStatusResponse" + } + } + } + }, + "400": { + "description": "Invalid desktop start request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "409": { + "description": "Desktop runtime is already transitioning", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "501": { + "description": "Desktop API unsupported on this platform", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "503": { + "description": "Desktop runtime could not be started", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/desktop/status": { + "get": { + "tags": [ + "v1" + ], + "summary": "Get desktop runtime status.", + "description": "Returns the current desktop runtime state, dependency status, active\ndisplay metadata, and supervised process information.", + "operationId": "get_v1_desktop_status", + "responses": { + "200": { + "description": "Desktop runtime status", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopStatusResponse" + } + } + } + }, + "401": { + "description": "Authentication required", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/v1/desktop/stop": { + "post": { + "tags": [ + "v1" + ], + "summary": "Stop the private desktop runtime.", + "description": "Terminates the managed openbox/Xvfb/dbus processes owned by the desktop\nruntime and returns the resulting status snapshot.", + "operationId": "post_v1_desktop_stop", + "responses": { + "200": { + "description": "Desktop runtime status after stop", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DesktopStatusResponse" + } + } + } + }, + "409": { + "description": "Desktop runtime is already transitioning", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, "/v1/fs/entries": { "get": { "tags": ["v1"], @@ -1934,6 +2650,390 @@ } } }, + "DesktopActionResponse": { + "type": "object", + "required": [ + "ok" + ], + "properties": { + "ok": { + "type": "boolean" + } + } + }, + "DesktopDisplayInfoResponse": { + "type": "object", + "required": [ + "display", + "resolution" + ], + "properties": { + "display": { + "type": "string" + }, + "resolution": { + "$ref": "#/components/schemas/DesktopResolution" + } + } + }, + "DesktopErrorInfo": { + "type": "object", + "required": [ + "code", + "message" + ], + "properties": { + "code": { + "type": "string" + }, + "message": { + "type": "string" + } + } + }, + "DesktopKeyboardPressRequest": { + "type": "object", + "required": [ + "key" + ], + "properties": { + "key": { + "type": "string" + } + } + }, + "DesktopKeyboardTypeRequest": { + "type": "object", + "required": [ + "text" + ], + "properties": { + "delayMs": { + "type": "integer", + "format": "int32", + "nullable": true, + "minimum": 0 + }, + "text": { + "type": "string" + } + } + }, + "DesktopMouseButton": { + "type": "string", + "enum": [ + "left", + "middle", + "right" + ] + }, + "DesktopMouseClickRequest": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "button": { + "allOf": [ + { + "$ref": "#/components/schemas/DesktopMouseButton" + } + ], + "nullable": true + }, + "clickCount": { + "type": "integer", + "format": "int32", + "nullable": true, + "minimum": 0 + }, + "x": { + "type": "integer", + "format": "int32" + }, + "y": { + "type": "integer", + "format": "int32" + } + } + }, + "DesktopMouseDragRequest": { + "type": "object", + "required": [ + "startX", + "startY", + "endX", + "endY" + ], + "properties": { + "button": { + "allOf": [ + { + "$ref": "#/components/schemas/DesktopMouseButton" + } + ], + "nullable": true + }, + "endX": { + "type": "integer", + "format": "int32" + }, + "endY": { + "type": "integer", + "format": "int32" + }, + "startX": { + "type": "integer", + "format": "int32" + }, + "startY": { + "type": "integer", + "format": "int32" + } + } + }, + "DesktopMouseMoveRequest": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "integer", + "format": "int32" + }, + "y": { + "type": "integer", + "format": "int32" + } + } + }, + "DesktopMousePositionResponse": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "screen": { + "type": "integer", + "format": "int32", + "nullable": true + }, + "window": { + "type": "string", + "nullable": true + }, + "x": { + "type": "integer", + "format": "int32" + }, + "y": { + "type": "integer", + "format": "int32" + } + } + }, + "DesktopMouseScrollRequest": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "deltaX": { + "type": "integer", + "format": "int32", + "nullable": true + }, + "deltaY": { + "type": "integer", + "format": "int32", + "nullable": true + }, + "x": { + "type": "integer", + "format": "int32" + }, + "y": { + "type": "integer", + "format": "int32" + } + } + }, + "DesktopProcessInfo": { + "type": "object", + "required": [ + "name", + "running" + ], + "properties": { + "logPath": { + "type": "string", + "nullable": true + }, + "name": { + "type": "string" + }, + "pid": { + "type": "integer", + "format": "int32", + "nullable": true, + "minimum": 0 + }, + "running": { + "type": "boolean" + } + } + }, + "DesktopRegionScreenshotQuery": { + "type": "object", + "required": [ + "x", + "y", + "width", + "height" + ], + "properties": { + "height": { + "type": "integer", + "format": "int32", + "minimum": 0 + }, + "width": { + "type": "integer", + "format": "int32", + "minimum": 0 + }, + "x": { + "type": "integer", + "format": "int32" + }, + "y": { + "type": "integer", + "format": "int32" + } + } + }, + "DesktopResolution": { + "type": "object", + "required": [ + "width", + "height" + ], + "properties": { + "dpi": { + "type": "integer", + "format": "int32", + "nullable": true, + "minimum": 0 + }, + "height": { + "type": "integer", + "format": "int32", + "minimum": 0 + }, + "width": { + "type": "integer", + "format": "int32", + "minimum": 0 + } + } + }, + "DesktopScreenshotQuery": { + "type": "object" + }, + "DesktopStartRequest": { + "type": "object", + "properties": { + "dpi": { + "type": "integer", + "format": "int32", + "nullable": true, + "minimum": 0 + }, + "height": { + "type": "integer", + "format": "int32", + "nullable": true, + "minimum": 0 + }, + "width": { + "type": "integer", + "format": "int32", + "nullable": true, + "minimum": 0 + } + } + }, + "DesktopState": { + "type": "string", + "enum": [ + "inactive", + "install_required", + "starting", + "active", + "stopping", + "failed" + ] + }, + "DesktopStatusResponse": { + "type": "object", + "required": [ + "state" + ], + "properties": { + "display": { + "type": "string", + "nullable": true + }, + "installCommand": { + "type": "string", + "nullable": true + }, + "lastError": { + "allOf": [ + { + "$ref": "#/components/schemas/DesktopErrorInfo" + } + ], + "nullable": true + }, + "missingDependencies": { + "type": "array", + "items": { + "type": "string" + } + }, + "processes": { + "type": "array", + "items": { + "$ref": "#/components/schemas/DesktopProcessInfo" + } + }, + "resolution": { + "allOf": [ + { + "$ref": "#/components/schemas/DesktopResolution" + } + ], + "nullable": true + }, + "runtimeLogPath": { + "type": "string", + "nullable": true + }, + "startedAt": { + "type": "string", + "nullable": true + }, + "state": { + "$ref": "#/components/schemas/DesktopState" + } + } + }, "ErrorType": { "type": "string", "enum": [ diff --git a/docs/quickstart.mdx b/docs/quickstart.mdx index 7b5beed..5e0af9b 100644 --- a/docs/quickstart.mdx +++ b/docs/quickstart.mdx @@ -226,6 +226,16 @@ icon: "rocket" If agents are not installed up front, they are lazily installed when creating a session. + + If you want to use `/v1/desktop/*`, install the desktop runtime packages first: + + ```bash + sandbox-agent install desktop --yes + ``` + + Then use `GET /v1/desktop/status` or `sdk.getDesktopStatus()` to verify the runtime is ready before calling desktop screenshot or input APIs. + + ```typescript import { SandboxAgent } from "sandbox-agent"; diff --git a/docs/sdk-overview.mdx b/docs/sdk-overview.mdx index fc4aee1..416cacf 100644 --- a/docs/sdk-overview.mdx +++ b/docs/sdk-overview.mdx @@ -190,6 +190,44 @@ const writeResult = await sdk.writeFsFile({ path: "./hello.txt" }, "hello"); console.log(health.status, agents.agents.length, entries.length, writeResult.path); ``` +## Desktop API + +The SDK also wraps the desktop host/runtime HTTP API. + +Install desktop dependencies first on Linux hosts: + +```bash +sandbox-agent install desktop --yes +``` + +Then query status, surface remediation if needed, and start the runtime: + +```ts +const status = await sdk.getDesktopStatus(); + +if (status.state === "install_required") { + console.log(status.installCommand); +} + +const started = await sdk.startDesktop({ + width: 1440, + height: 900, + dpi: 96, +}); + +const screenshot = await sdk.takeDesktopScreenshot(); +const displayInfo = await sdk.getDesktopDisplayInfo(); + +await sdk.moveDesktopMouse({ x: 400, y: 300 }); +await sdk.clickDesktop({ x: 400, y: 300, button: "left", clickCount: 1 }); +await sdk.typeDesktopText({ text: "hello world", delayMs: 10 }); +await sdk.pressDesktopKey({ key: "ctrl+l" }); + +await sdk.stopDesktop(); +``` + +Screenshot helpers return `Uint8Array` PNG bytes. The SDK does not attempt to install OS packages remotely; callers should surface `missingDependencies` and `installCommand` from `getDesktopStatus()`. + ## Error handling ```ts diff --git a/frontend/packages/inspector/index.html b/frontend/packages/inspector/index.html index 5893717..c5c1ab7 100644 --- a/frontend/packages/inspector/index.html +++ b/frontend/packages/inspector/index.html @@ -2889,6 +2889,94 @@ gap: 20px; } + .desktop-panel { + display: flex; + flex-direction: column; + gap: 16px; + } + + .desktop-state-grid { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 12px; + margin-bottom: 12px; + } + + .desktop-start-controls { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 10px; + } + + .desktop-input-group { + display: flex; + flex-direction: column; + gap: 4px; + } + + .desktop-chip-list { + display: flex; + flex-wrap: wrap; + gap: 8px; + } + + .desktop-command { + margin-top: 6px; + padding: 8px 10px; + border-radius: var(--radius); + border: 1px solid var(--border); + background: var(--surface); + overflow-x: auto; + } + + .desktop-diagnostic-block + .desktop-diagnostic-block { + margin-top: 14px; + } + + .desktop-process-list { + display: flex; + flex-direction: column; + gap: 10px; + margin-top: 8px; + } + + .desktop-process-item { + padding: 10px; + border-radius: var(--radius); + border: 1px solid var(--border); + background: var(--surface); + display: flex; + flex-direction: column; + gap: 4px; + } + + .desktop-screenshot-empty { + padding: 18px; + border: 1px dashed var(--border); + border-radius: var(--radius); + color: var(--muted); + background: var(--surface); + text-align: center; + } + + .desktop-screenshot-frame { + border-radius: calc(var(--radius) + 2px); + overflow: hidden; + border: 1px solid var(--border); + background: + linear-gradient(135deg, rgba(15, 23, 42, 0.9), rgba(30, 41, 59, 0.92)), + radial-gradient(circle at top right, rgba(56, 189, 248, 0.12), transparent 40%); + padding: 10px; + } + + .desktop-screenshot-image { + display: block; + width: 100%; + height: auto; + border-radius: var(--radius); + background: rgba(0, 0, 0, 0.24); + } + .processes-section { display: flex; flex-direction: column; @@ -3551,6 +3639,11 @@ grid-template-columns: 1fr; } + .desktop-state-grid, + .desktop-start-controls { + grid-template-columns: 1fr; + } + .session-sidebar { display: none; } diff --git a/frontend/packages/inspector/package.json b/frontend/packages/inspector/package.json index 9671ecb..28cd0c7 100644 --- a/frontend/packages/inspector/package.json +++ b/frontend/packages/inspector/package.json @@ -18,6 +18,7 @@ "@types/react-dom": "^18.3.0", "@vitejs/plugin-react": "^4.3.1", "fake-indexeddb": "^6.2.4", + "jsdom": "^26.1.0", "typescript": "^5.7.3", "vite": "^5.4.7", "vitest": "^3.0.0" diff --git a/frontend/packages/inspector/src/components/debug/DebugPanel.tsx b/frontend/packages/inspector/src/components/debug/DebugPanel.tsx index 879ab96..5398fe6 100644 --- a/frontend/packages/inspector/src/components/debug/DebugPanel.tsx +++ b/frontend/packages/inspector/src/components/debug/DebugPanel.tsx @@ -1,4 +1,4 @@ -import { ChevronLeft, ChevronRight, Cloud, Play, PlayCircle, Server, Terminal, Wrench } from "lucide-react"; +import { ChevronLeft, ChevronRight, Cloud, Monitor, Play, PlayCircle, Server, Terminal, Wrench } from "lucide-react"; import type { AgentInfo, SandboxAgent, SessionEvent } from "sandbox-agent"; type AgentModeInfo = { id: string; name: string; description: string }; @@ -9,9 +9,10 @@ import ProcessesTab from "./ProcessesTab"; import ProcessRunTab from "./ProcessRunTab"; import SkillsTab from "./SkillsTab"; import RequestLogTab from "./RequestLogTab"; +import DesktopTab from "./DesktopTab"; import type { RequestLog } from "../../types/requestLog"; -export type DebugTab = "log" | "events" | "agents" | "mcp" | "skills" | "processes" | "run-process"; +export type DebugTab = "log" | "events" | "agents" | "desktop" | "mcp" | "skills" | "processes" | "run-process"; const DebugPanel = ({ debugTab, @@ -75,6 +76,10 @@ const DebugPanel = ({ Agents + + + + + {error &&
{error}
} + {screenshotError &&
{screenshotError}
} + +
+
+ + + Desktop Runtime + + + {status?.state ?? "unknown"} + +
+ +
+
+
Display
+
{status?.display ?? "Not assigned"}
+
+
+
Resolution
+
{resolutionLabel}
+
+
+
Started
+
{formatStartedAt(status?.startedAt)}
+
+
+ +
+
+ + setWidth(event.target.value)} + inputMode="numeric" + /> +
+
+ + setHeight(event.target.value)} + inputMode="numeric" + /> +
+
+ + setDpi(event.target.value)} + inputMode="numeric" + /> +
+
+ +
+ + +
+
+ + {status?.missingDependencies && status.missingDependencies.length > 0 && ( +
+
+ Missing Dependencies +
+
+ {status.missingDependencies.map((dependency) => ( + {dependency} + ))} +
+ {status.installCommand && ( + <> +
Install command
+
{status.installCommand}
+ + )} +
+ )} + + {(status?.lastError || status?.runtimeLogPath || (status?.processes?.length ?? 0) > 0) && ( +
+
+ Diagnostics +
+ {status?.lastError && ( +
+
Last error
+
{status.lastError.code}
+
{status.lastError.message}
+
+ )} + {status?.runtimeLogPath && ( +
+
Runtime log
+
{status.runtimeLogPath}
+
+ )} + {status?.processes && status.processes.length > 0 && ( +
+
Processes
+
+ {status.processes.map((process) => ( +
+
+ {process.name} + + {process.running ? "running" : "stopped"} + +
+
+ {process.pid ? `pid ${process.pid}` : "no pid"} +
+ {process.logPath &&
{process.logPath}
} +
+ ))} +
+
+ )} +
+ )} + +
+
+ Latest Screenshot + {status?.state === "active" ? ( + Manual refresh only + ) : null} +
+ + {loading ?
Loading...
: null} + {!loading && !screenshotUrl && ( +
+ {status?.state === "active" + ? "No screenshot loaded yet." + : "Start the desktop runtime to capture a screenshot."} +
+ )} + {screenshotUrl && ( +
+ Desktop screenshot +
+ )} +
+ + ); +}; + +export default DesktopTab; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 14f8572..fe82ce0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,7 +29,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@24.10.9)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@24.10.9)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) examples/boxlite: dependencies: @@ -94,7 +94,7 @@ importers: version: 6.4.1(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) wrangler: specifier: latest version: 4.73.0(@cloudflare/workers-types@4.20260313.1) @@ -122,7 +122,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) examples/daytona: dependencies: @@ -172,7 +172,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) examples/e2b: dependencies: @@ -197,7 +197,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) examples/file-system: dependencies: @@ -277,6 +277,31 @@ importers: specifier: latest version: 5.9.3 + examples/modal: + dependencies: + '@sandbox-agent/example-shared': + specifier: workspace:* + version: link:../shared + modal: + specifier: latest + version: 0.7.3 + sandbox-agent: + specifier: workspace:* + version: link:../../sdks/typescript + devDependencies: + '@types/node': + specifier: latest + version: 25.5.0 + tsx: + specifier: latest + version: 4.21.0 + typescript: + specifier: latest + version: 5.9.3 + vitest: + specifier: ^3.0.0 + version: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) + examples/permissions: dependencies: commander: @@ -296,31 +321,6 @@ importers: specifier: latest version: 5.9.3 - examples/modal: - dependencies: - '@sandbox-agent/example-shared': - specifier: workspace:* - version: link:../shared - modal: - specifier: latest - version: 0.7.1 - sandbox-agent: - specifier: workspace:* - version: link:../../sdks/typescript - devDependencies: - '@types/node': - specifier: latest - version: 25.3.0 - tsx: - specifier: latest - version: 4.21.0 - typescript: - specifier: latest - version: 5.9.3 - vitest: - specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@25.3.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) - examples/persist-memory: dependencies: '@sandbox-agent/example-shared': @@ -473,7 +473,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) foundry/packages/backend: dependencies: @@ -497,7 +497,7 @@ importers: version: link:../../../sdks/persist-rivet better-auth: specifier: ^1.5.5 - version: 1.5.5(@cloudflare/workers-types@4.20260313.1)(drizzle-kit@0.31.9)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260313.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0))(pg@8.20.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(solid-js@1.9.11)(vitest@3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2)) + version: 1.5.5(@cloudflare/workers-types@4.20260313.1)(drizzle-kit@0.31.9)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260313.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0))(pg@8.20.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(solid-js@1.9.11)(vitest@3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2)) dockerode: specifier: ^4.0.9 version: 4.0.9 @@ -668,6 +668,9 @@ importers: fake-indexeddb: specifier: ^6.2.4 version: 6.2.5 + jsdom: + specifier: ^26.1.0 + version: 26.1.0 sandbox-agent: specifier: workspace:* version: link:../../../sdks/typescript @@ -679,7 +682,7 @@ importers: version: 5.4.21(@types/node@25.5.0) vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) frontend/packages/website: dependencies: @@ -810,7 +813,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) sdks/cli: dependencies: @@ -836,7 +839,7 @@ importers: devDependencies: vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) sdks/cli-shared: devDependencies: @@ -884,7 +887,7 @@ importers: devDependencies: vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) sdks/gigacode/platforms/darwin-arm64: {} @@ -916,7 +919,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) sdks/persist-postgres: dependencies: @@ -941,7 +944,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) sdks/persist-rivet: dependencies: @@ -960,7 +963,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) sdks/persist-sqlite: dependencies: @@ -985,7 +988,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) sdks/react: dependencies: @@ -1042,7 +1045,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) ws: specifier: ^8.19.0 version: 8.19.0 @@ -1061,7 +1064,7 @@ importers: version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) packages: @@ -1078,6 +1081,9 @@ packages: resolution: {integrity: sha512-FSEVWXvwroExDXUu8qV6Wqp2X3D1nJ0Li4LFymCyvCVrm7I3lNfG0zZWSWvGU1RE7891eTnFTyh31L3igOwNKQ==} hasBin: true + '@asamuzakjp/css-color@3.2.0': + resolution: {integrity: sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw==} + '@asteasolutions/zod-to-openapi@8.4.3': resolution: {integrity: sha512-lwfMTN7kDbFDwMniYZUebiGGHxVGBw9ZSI4IBYjm6Ey22Kd5z/fsQb2k+Okr8WMbCCC553vi/ZM9utl5/XcvuQ==} peerDependencies: @@ -1682,6 +1688,34 @@ packages: resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} + '@csstools/color-helpers@5.1.0': + resolution: {integrity: sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA==} + engines: {node: '>=18'} + + '@csstools/css-calc@2.1.4': + resolution: {integrity: sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ==} + engines: {node: '>=18'} + peerDependencies: + '@csstools/css-parser-algorithms': ^3.0.5 + '@csstools/css-tokenizer': ^3.0.4 + + '@csstools/css-color-parser@3.1.0': + resolution: {integrity: sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA==} + engines: {node: '>=18'} + peerDependencies: + '@csstools/css-parser-algorithms': ^3.0.5 + '@csstools/css-tokenizer': ^3.0.4 + + '@csstools/css-parser-algorithms@3.0.5': + resolution: {integrity: sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ==} + engines: {node: '>=18'} + peerDependencies: + '@csstools/css-tokenizer': ^3.0.4 + + '@csstools/css-tokenizer@3.0.4': + resolution: {integrity: sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw==} + engines: {node: '>=18'} + '@date-io/core@2.17.0': resolution: {integrity: sha512-+EQE8xZhRM/hsY0CDTVyayMDDY5ihc4MqXCrPxooKw19yAzUIC6uUqsZeaOFNL9YKTNxYKrJP5DFgE8o5xRCOw==} @@ -3713,6 +3747,10 @@ packages: acp-http-client@0.3.2: resolution: {integrity: sha512-btRUDXAA9BlcTQURsJogdWthoXsKOnMeFhtYlEYQxgt0vq7H6xMfMrewlIgFjRXgRTbru4Fre2T6wS/amTTyjQ==} + agent-base@7.1.4: + resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} + engines: {node: '>= 14'} + aggregate-error@5.0.0: resolution: {integrity: sha512-gOsf2YwSlleG6IjRYG2A7k0HmBMEo6qVNk9Bp/EaLgAJT5ngH6PXbqa4ItvnEwCm/velL5jAnQgsHsWnjhGmvw==} engines: {node: '>=18'} @@ -4245,6 +4283,10 @@ packages: resolution: {integrity: sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ==} engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0, npm: '>=7.0.0'} + cssstyle@4.6.0: + resolution: {integrity: sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg==} + engines: {node: '>=18'} + csstype@2.6.11: resolution: {integrity: sha512-l8YyEC9NBkSm783PFTvh0FmJy7s5pFKrDp49ZL7zBGX3fWkO+N4EEyan1qqp8cwPLDcD0OSdyY6hAMoxp34JFw==} @@ -4378,6 +4420,10 @@ packages: resolution: {integrity: sha512-e1U46jVP+w7Iut8Jt8ri1YsPOvFpg46k+K8TpCb0P+zjCkjkPnV7WzfDJzMHy1LnA+wj5pLT1wjO901gLXeEhA==} engines: {node: '>=12'} + data-urls@5.0.0: + resolution: {integrity: sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==} + engines: {node: '>=18'} + date-fns-tz@1.3.8: resolution: {integrity: sha512-qwNXUFtMHTTU6CFSFjoJ80W8Fzzp24LntbjFFBgL/faqds4e5mo9mftoRLgr3Vi1trISsg4awSpYVsOQCRnapQ==} peerDependencies: @@ -4396,6 +4442,9 @@ packages: supports-color: optional: true + decimal.js@10.6.0: + resolution: {integrity: sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==} + decode-named-character-reference@1.3.0: resolution: {integrity: sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==} @@ -5024,6 +5073,10 @@ packages: resolution: {integrity: sha512-gJnaDHXKDayjt8ue0n8Gs0A007yKXj4Xzb8+cNjZeYsSzzwKc0Lr+OZgYwVfB0pHfUs17EPoLvrOsEaJ9mj+Tg==} engines: {node: '>=16.9.0'} + html-encoding-sniffer@4.0.0: + resolution: {integrity: sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==} + engines: {node: '>=18'} + html-escaper@3.0.3: resolution: {integrity: sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==} @@ -5037,6 +5090,14 @@ packages: resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} engines: {node: '>= 0.8'} + http-proxy-agent@7.0.2: + resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} + engines: {node: '>= 14'} + + https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + human-signals@3.0.1: resolution: {integrity: sha512-rQLskxnM/5OCldHo+wNXbpVgDn5A17CUoKX+7Sokwaknlq7CdSnphy0W39GU8dw59XiCXmFXDg4fRuckQRKewQ==} engines: {node: '>=12.20.0'} @@ -5146,6 +5207,9 @@ packages: resolution: {integrity: sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==} engines: {node: '>=12'} + is-potential-custom-element-name@1.0.1: + resolution: {integrity: sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==} + is-promise@4.0.0: resolution: {integrity: sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==} @@ -5209,6 +5273,15 @@ packages: resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} hasBin: true + jsdom@26.1.0: + resolution: {integrity: sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg==} + engines: {node: '>=18'} + peerDependencies: + canvas: ^3.0.0 + peerDependenciesMeta: + canvas: + optional: true + jsesc@3.1.0: resolution: {integrity: sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==} engines: {node: '>=6'} @@ -5584,8 +5657,11 @@ packages: mlly@1.8.0: resolution: {integrity: sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g==} - modal@0.7.1: - resolution: {integrity: sha512-WFn5mfVD7BbdNytqDODjKXG+RkF4bubTKiu7gZvq/JITcLIU1JWYnZQSJ41cE1TlrBlxFADSx8d7Q2AXF1GT+A==} + mockdate@2.0.5: + resolution: {integrity: sha512-ST0PnThzWKcgSLyc+ugLVql45PvESt3Ul/wrdV/OPc/6Pr8dbLAIJsN1cIp41FLzbN+srVTNIRn+5Cju0nyV6A==} + + modal@0.7.3: + resolution: {integrity: sha512-4CliqNF15sZPBGpSoCj5Y9fd8fTp1ONrBLIJiC4amm/Qzc1rn8CH45SVzSu+1DokHCIRiZqQ1xMhRKpDvDCkBw==} module-details-from-path@1.0.4: resolution: {integrity: sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==} @@ -5676,6 +5752,9 @@ packages: nth-check@2.1.1: resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} + nwsapi@2.2.23: + resolution: {integrity: sha512-7wfH4sLbt4M0gCDzGE6vzQBo0bfTKjU7Sfpqy/7gs1qBfYz2vEJH6vXcBKpO3+6Yu1telwd0t9HpyOoLEQQbIQ==} + object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -6058,6 +6137,10 @@ packages: pump@3.0.3: resolution: {integrity: sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==} + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + qs@6.14.1: resolution: {integrity: sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==} engines: {node: '>=0.6'} @@ -6400,6 +6483,9 @@ packages: resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==} engines: {node: '>= 18'} + rrweb-cssom@0.8.0: + resolution: {integrity: sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==} + run-parallel@1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} @@ -6423,6 +6509,10 @@ packages: resolution: {integrity: sha512-1n3r/tGXO6b6VXMdFT54SHzT9ytu9yr7TaELowdYpMqY/Ao7EnlQGmAQ1+RatX7Tkkdm6hONI2owqNx2aZj5Sw==} engines: {node: '>=11.0.0'} + saxes@6.0.0: + resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==} + engines: {node: '>=v12.22.7'} + scheduler@0.23.2: resolution: {integrity: sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==} @@ -6663,6 +6753,9 @@ packages: engines: {node: '>=16'} hasBin: true + symbol-tree@3.2.4: + resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==} + tailwindcss@3.4.19: resolution: {integrity: sha512-3ofp+LL8E+pK/JuPLPggVAIaEuhvIz4qNcf3nA1Xn2o/7fb7s/TYpHhwGDv1ZU3PkBluUVaF8PyCHcm48cKLWQ==} engines: {node: '>=14.0.0'} @@ -6747,6 +6840,13 @@ packages: resolution: {integrity: sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==} engines: {node: '>=14.0.0'} + tldts-core@6.1.86: + resolution: {integrity: sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==} + + tldts@6.1.86: + resolution: {integrity: sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==} + hasBin: true + to-regex-range@5.0.1: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} @@ -6755,6 +6855,14 @@ packages: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} + tough-cookie@5.1.2: + resolution: {integrity: sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==} + engines: {node: '>=16'} + + tr46@5.1.1: + resolution: {integrity: sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==} + engines: {node: '>=18'} + tree-kill@1.2.2: resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} hasBin: true @@ -7236,12 +7344,33 @@ packages: vt-pbf@3.1.3: resolution: {integrity: sha512-2LzDFzt0mZKZ9IpVF2r69G9bXaP2Q2sArJCmcCgvfTdCCZzSyz4aCLoQyUilu37Ll56tCblIZrXFIjNUpGIlmA==} + w3c-xmlserializer@5.0.0: + resolution: {integrity: sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==} + engines: {node: '>=18'} + warning@4.0.3: resolution: {integrity: sha512-rpJyN222KWIvHJ/F53XSZv0Zl/accqHR8et1kpaMTD/fLCRxtV8iX8czMzY7sVZupTI3zcUTg8eycS2kNF9l6w==} web-namespaces@2.0.1: resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} + webidl-conversions@7.0.0: + resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} + engines: {node: '>=12'} + + whatwg-encoding@3.1.1: + resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==} + engines: {node: '>=18'} + deprecated: Use @exodus/bytes instead for a more spec-conformant and faster implementation + + whatwg-mimetype@4.0.0: + resolution: {integrity: sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==} + engines: {node: '>=18'} + + whatwg-url@14.2.0: + resolution: {integrity: sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==} + engines: {node: '>=18'} + which-pm-runs@1.1.0: resolution: {integrity: sha512-n1brCuqClxfFfq/Rb0ICg9giSZqCS+pLtccdag6C2HyufBrh3fBOiy9nb6ggRMvWOVH5GrdJskj5iGTZNxd7SA==} engines: {node: '>=4'} @@ -7322,6 +7451,13 @@ packages: resolution: {integrity: sha512-sqMMuL1rc0FmMBOzCpd0yuy9trqF2yTTVe+E9ogwCSWQCdDEtQUwrZPT6AxqtsFGRNxycgncbP/xmOOSPw5ZUw==} engines: {node: '>= 6.0'} + xml-name-validator@5.0.0: + resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==} + engines: {node: '>=18'} + + xmlchars@2.2.0: + resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} + xtend@4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} @@ -7404,6 +7540,14 @@ snapshots: '@antfu/ni@0.23.2': {} + '@asamuzakjp/css-color@3.2.0': + dependencies: + '@csstools/css-calc': 2.1.4(@csstools/css-parser-algorithms@3.0.5(@csstools/css-tokenizer@3.0.4))(@csstools/css-tokenizer@3.0.4) + '@csstools/css-color-parser': 3.1.0(@csstools/css-parser-algorithms@3.0.5(@csstools/css-tokenizer@3.0.4))(@csstools/css-tokenizer@3.0.4) + '@csstools/css-parser-algorithms': 3.0.5(@csstools/css-tokenizer@3.0.4) + '@csstools/css-tokenizer': 3.0.4 + lru-cache: 10.4.3 + '@asteasolutions/zod-to-openapi@8.4.3(zod@4.3.6)': dependencies: openapi3-ts: 4.5.0 @@ -8398,6 +8542,26 @@ snapshots: dependencies: '@jridgewell/trace-mapping': 0.3.9 + '@csstools/color-helpers@5.1.0': {} + + '@csstools/css-calc@2.1.4(@csstools/css-parser-algorithms@3.0.5(@csstools/css-tokenizer@3.0.4))(@csstools/css-tokenizer@3.0.4)': + dependencies: + '@csstools/css-parser-algorithms': 3.0.5(@csstools/css-tokenizer@3.0.4) + '@csstools/css-tokenizer': 3.0.4 + + '@csstools/css-color-parser@3.1.0(@csstools/css-parser-algorithms@3.0.5(@csstools/css-tokenizer@3.0.4))(@csstools/css-tokenizer@3.0.4)': + dependencies: + '@csstools/color-helpers': 5.1.0 + '@csstools/css-calc': 2.1.4(@csstools/css-parser-algorithms@3.0.5(@csstools/css-tokenizer@3.0.4))(@csstools/css-tokenizer@3.0.4) + '@csstools/css-parser-algorithms': 3.0.5(@csstools/css-tokenizer@3.0.4) + '@csstools/css-tokenizer': 3.0.4 + + '@csstools/css-parser-algorithms@3.0.5(@csstools/css-tokenizer@3.0.4)': + dependencies: + '@csstools/css-tokenizer': 3.0.4 + + '@csstools/css-tokenizer@3.0.4': {} + '@date-io/core@2.17.0': {} '@date-io/date-fns@2.17.0(date-fns@2.30.0)': @@ -10112,7 +10276,7 @@ snapshots: '@types/better-sqlite3@7.6.13': dependencies: - '@types/node': 25.3.0 + '@types/node': 24.10.9 '@types/bun@1.3.10': dependencies: @@ -10180,7 +10344,7 @@ snapshots: '@types/pg@8.16.0': dependencies: - '@types/node': 25.3.0 + '@types/node': 24.10.9 pg-protocol: 1.11.0 pg-types: 2.2.0 @@ -10209,7 +10373,7 @@ snapshots: '@types/sax@1.2.7': dependencies: - '@types/node': 25.3.0 + '@types/node': 24.10.9 '@types/semver@7.7.1': {} @@ -10286,21 +10450,29 @@ snapshots: chai: 5.3.3 tinyrainbow: 2.0.0 - '@vitest/mocker@3.2.4(vite@6.4.1(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2))': + '@vitest/mocker@3.2.4(vite@7.3.1(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2))': dependencies: '@vitest/spy': 3.2.4 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 6.4.1(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + vite: 7.3.1(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) - '@vitest/mocker@3.2.4(vite@6.4.1(@types/node@25.3.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2))': + '@vitest/mocker@3.2.4(vite@7.3.1(@types/node@24.10.9)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2))': dependencies: '@vitest/spy': 3.2.4 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 6.4.1(@types/node@25.3.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + vite: 7.3.1(@types/node@24.10.9)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + + '@vitest/mocker@3.2.4(vite@7.3.1(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2))': + dependencies: + '@vitest/spy': 3.2.4 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 7.3.1(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) '@vitest/pretty-format@3.2.4': dependencies: @@ -10347,6 +10519,8 @@ snapshots: transitivePeerDependencies: - zod + agent-base@7.1.4: {} + aggregate-error@5.0.0: dependencies: clean-stack: 5.3.0 @@ -10593,7 +10767,7 @@ snapshots: dependencies: tweetnacl: 0.14.5 - better-auth@1.5.5(@cloudflare/workers-types@4.20260313.1)(drizzle-kit@0.31.9)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260313.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0))(pg@8.20.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(solid-js@1.9.11)(vitest@3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2)): + better-auth@1.5.5(@cloudflare/workers-types@4.20260313.1)(drizzle-kit@0.31.9)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260313.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0))(pg@8.20.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(solid-js@1.9.11)(vitest@3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2)): dependencies: '@better-auth/core': 1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1) '@better-auth/drizzle-adapter': 1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260313.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0)) @@ -10619,7 +10793,7 @@ snapshots: react: 19.2.4 react-dom: 19.2.4(react@19.2.4) solid-js: 1.9.11 - vitest: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + vitest: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) transitivePeerDependencies: - '@cloudflare/workers-types' @@ -10949,6 +11123,11 @@ snapshots: dependencies: css-tree: 2.2.1 + cssstyle@4.6.0: + dependencies: + '@asamuzakjp/css-color': 3.2.0 + rrweb-cssom: 0.8.0 + csstype@2.6.11: {} csstype@3.2.3: {} @@ -11105,6 +11284,11 @@ snapshots: d3-transition: 3.0.1(d3-selection@3.0.0) d3-zoom: 3.0.0 + data-urls@5.0.0: + dependencies: + whatwg-mimetype: 4.0.0 + whatwg-url: 14.2.0 + date-fns-tz@1.3.8(date-fns@2.30.0): dependencies: date-fns: 2.30.0 @@ -11117,6 +11301,8 @@ snapshots: dependencies: ms: 2.1.3 + decimal.js@10.6.0: {} + decode-named-character-reference@1.3.0: dependencies: character-entities: 2.0.2 @@ -11857,6 +12043,10 @@ snapshots: hono@4.12.2: {} + html-encoding-sniffer@4.0.0: + dependencies: + whatwg-encoding: 3.1.1 + html-escaper@3.0.3: {} html-void-elements@3.0.0: {} @@ -11871,6 +12061,20 @@ snapshots: statuses: 2.0.2 toidentifier: 1.0.1 + http-proxy-agent@7.0.2: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + https-proxy-agent@7.0.6: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + human-signals@3.0.1: {} human-signals@5.0.0: {} @@ -11950,6 +12154,8 @@ snapshots: is-plain-obj@4.1.0: {} + is-potential-custom-element-name@1.0.1: {} + is-promise@4.0.0: {} is-stream@3.0.0: {} @@ -11996,6 +12202,33 @@ snapshots: dependencies: argparse: 2.0.1 + jsdom@26.1.0: + dependencies: + cssstyle: 4.6.0 + data-urls: 5.0.0 + decimal.js: 10.6.0 + html-encoding-sniffer: 4.0.0 + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + is-potential-custom-element-name: 1.0.1 + nwsapi: 2.2.23 + parse5: 7.3.0 + rrweb-cssom: 0.8.0 + saxes: 6.0.0 + symbol-tree: 3.2.4 + tough-cookie: 5.1.2 + w3c-xmlserializer: 5.0.0 + webidl-conversions: 7.0.0 + whatwg-encoding: 3.1.1 + whatwg-mimetype: 4.0.0 + whatwg-url: 14.2.0 + ws: 8.19.0 + xml-name-validator: 5.0.0 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + jsesc@3.1.0: {} json-schema-traverse@1.0.0: {} @@ -12533,9 +12766,11 @@ snapshots: pkg-types: 1.3.1 ufo: 1.6.3 - modal@0.7.1: + mockdate@2.0.5: {} + + modal@0.7.3: dependencies: - cbor-x: 1.6.0 + cbor-x: 1.6.3 long: 5.3.2 nice-grpc: 2.1.14 protobufjs: 7.5.4 @@ -12618,6 +12853,8 @@ snapshots: dependencies: boolbase: 1.0.0 + nwsapi@2.2.23: {} + object-assign@4.1.1: {} object-hash@3.0.0: {} @@ -13012,6 +13249,8 @@ snapshots: end-of-stream: 1.4.5 once: 1.4.0 + punycode@2.3.1: {} + qs@6.14.1: dependencies: side-channel: 1.1.0 @@ -13450,6 +13689,8 @@ snapshots: transitivePeerDependencies: - supports-color + rrweb-cssom@0.8.0: {} + run-parallel@1.2.0: dependencies: queue-microtask: 1.2.3 @@ -13473,6 +13714,10 @@ snapshots: sax@1.4.4: {} + saxes@6.0.0: + dependencies: + xmlchars: 2.2.0 + scheduler@0.23.2: dependencies: loose-envify: 1.4.0 @@ -13774,6 +14019,8 @@ snapshots: picocolors: 1.1.1 sax: 1.4.4 + symbol-tree@3.2.4: {} + tailwindcss@3.4.19(tsx@4.21.0)(yaml@2.8.2): dependencies: '@alloc/quick-lru': 5.2.0 @@ -13893,12 +14140,26 @@ snapshots: tinyspy@4.0.4: {} + tldts-core@6.1.86: {} + + tldts@6.1.86: + dependencies: + tldts-core: 6.1.86 + to-regex-range@5.0.1: dependencies: is-number: 7.0.0 toidentifier@1.0.1: {} + tough-cookie@5.1.2: + dependencies: + tldts: 6.1.86 + + tr46@5.1.1: + dependencies: + punycode: 2.3.1 + tree-kill@1.2.2: {} trim-lines@3.0.1: {} @@ -14225,15 +14486,6 @@ snapshots: - tsx - yaml - vite@5.4.21(@types/node@25.3.0): - dependencies: - esbuild: 0.21.5 - postcss: 8.5.6 - rollup: 4.56.0 - optionalDependencies: - '@types/node': 24.10.9 - fsevents: 2.3.3 - vite@5.4.21(@types/node@25.5.0): dependencies: esbuild: 0.21.5 @@ -14307,11 +14559,11 @@ snapshots: optionalDependencies: vite: 6.4.1(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) - vitest@3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2): + vitest@3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2): dependencies: '@types/chai': 5.2.3 '@vitest/expect': 3.2.4 - '@vitest/mocker': 3.2.4(vite@6.4.1(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2)) + '@vitest/mocker': 3.2.4(vite@7.3.1(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2)) '@vitest/pretty-format': 3.2.4 '@vitest/runner': 3.2.4 '@vitest/snapshot': 3.2.4 @@ -14329,12 +14581,13 @@ snapshots: tinyglobby: 0.2.15 tinypool: 1.1.1 tinyrainbow: 2.0.0 - vite: 6.4.1(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + vite: 7.3.1(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) vite-node: 3.2.4(@types/node@22.19.7)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) why-is-node-running: 2.3.0 optionalDependencies: '@types/debug': 4.1.12 '@types/node': 22.19.7 + jsdom: 26.1.0 transitivePeerDependencies: - jiti - less @@ -14349,11 +14602,11 @@ snapshots: - tsx - yaml - vitest@3.2.4(@types/debug@4.1.12)(@types/node@24.10.9)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2): + vitest@3.2.4(@types/debug@4.1.12)(@types/node@24.10.9)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2): dependencies: '@types/chai': 5.2.3 '@vitest/expect': 3.2.4 - '@vitest/mocker': 3.2.4(vite@5.4.21(@types/node@24.10.9)) + '@vitest/mocker': 3.2.4(vite@7.3.1(@types/node@24.10.9)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2)) '@vitest/pretty-format': 3.2.4 '@vitest/runner': 3.2.4 '@vitest/snapshot': 3.2.4 @@ -14371,12 +14624,13 @@ snapshots: tinyglobby: 0.2.15 tinypool: 1.1.1 tinyrainbow: 2.0.0 - vite: 5.4.21(@types/node@24.10.9) + vite: 7.3.1(@types/node@24.10.9)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) vite-node: 3.2.4(@types/node@24.10.9)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) why-is-node-running: 2.3.0 optionalDependencies: '@types/debug': 4.1.12 '@types/node': 24.10.9 + jsdom: 26.1.0 transitivePeerDependencies: - jiti - less @@ -14391,11 +14645,11 @@ snapshots: - tsx - yaml - vitest@3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2): + vitest@3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2): dependencies: '@types/chai': 5.2.3 '@vitest/expect': 3.2.4 - '@vitest/mocker': 3.2.4(vite@6.4.1(@types/node@25.3.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2)) + '@vitest/mocker': 3.2.4(vite@7.3.1(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2)) '@vitest/pretty-format': 3.2.4 '@vitest/runner': 3.2.4 '@vitest/snapshot': 3.2.4 @@ -14413,12 +14667,13 @@ snapshots: tinyglobby: 0.2.15 tinypool: 1.1.1 tinyrainbow: 2.0.0 - vite: 6.4.1(@types/node@25.3.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) - vite-node: 3.2.4(@types/node@25.3.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + vite: 7.3.1(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) + vite-node: 3.2.4(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2) why-is-node-running: 2.3.0 optionalDependencies: '@types/debug': 4.1.12 '@types/node': 25.5.0 + jsdom: 26.1.0 transitivePeerDependencies: - jiti - less @@ -14443,12 +14698,29 @@ snapshots: '@mapbox/vector-tile': 1.3.1 pbf: 3.3.0 + w3c-xmlserializer@5.0.0: + dependencies: + xml-name-validator: 5.0.0 + warning@4.0.3: dependencies: loose-envify: 1.4.0 web-namespaces@2.0.1: {} + webidl-conversions@7.0.0: {} + + whatwg-encoding@3.1.1: + dependencies: + iconv-lite: 0.6.3 + + whatwg-mimetype@4.0.0: {} + + whatwg-url@14.2.0: + dependencies: + tr46: 5.1.1 + webidl-conversions: 7.0.0 + which-pm-runs@1.1.0: {} which@2.0.2: @@ -14521,6 +14793,10 @@ snapshots: dependencies: os-paths: 4.4.0 + xml-name-validator@5.0.0: {} + + xmlchars@2.2.0: {} + xtend@4.0.2: {} xxhash-wasm@1.1.0: {} diff --git a/research/acp/friction.md b/research/acp/friction.md index 983b966..e5273b8 100644 --- a/research/acp/friction.md +++ b/research/acp/friction.md @@ -277,3 +277,13 @@ Update this file continuously during the migration. - Owner: Unassigned. - Status: resolved - Links: `sdks/acp-http-client/src/index.ts`, `sdks/acp-http-client/tests/smoke.test.ts`, `sdks/typescript/tests/integration.test.ts` + +- Date: 2026-03-07 +- Area: Desktop host/runtime API boundary +- Issue: Desktop automation needed screenshot/input/file-transfer-like host capabilities, but routing it through ACP would have mixed agent protocol semantics with host-owned runtime control and binary payloads. +- Impact: A desktop feature built as ACP methods would blur the division between agent/session behavior and Sandbox Agent host/runtime APIs, and would complicate binary screenshot transport. +- Proposed direction: Ship desktop as first-party HTTP endpoints under `/v1/desktop/*`, keep health/install/remediation in the server runtime, and expose the feature through the SDK and inspector without ACP extension methods. +- Decision: Accepted and implemented for phase one. +- Owner: Unassigned. +- Status: resolved +- Links: `server/packages/sandbox-agent/src/router.rs`, `server/packages/sandbox-agent/src/desktop_runtime.rs`, `sdks/typescript/src/client.ts`, `frontend/packages/inspector/src/components/debug/DesktopTab.tsx` diff --git a/sdks/typescript/src/client.ts b/sdks/typescript/src/client.ts index 9945c0a..12b88ce 100644 --- a/sdks/typescript/src/client.ts +++ b/sdks/typescript/src/client.ts @@ -29,6 +29,19 @@ import { type AgentInstallRequest, type AgentInstallResponse, type AgentListResponse, + type DesktopActionResponse, + type DesktopDisplayInfoResponse, + type DesktopKeyboardPressRequest, + type DesktopKeyboardTypeRequest, + type DesktopMouseClickRequest, + type DesktopMouseDragRequest, + type DesktopMouseMoveRequest, + type DesktopMousePositionResponse, + type DesktopMouseScrollRequest, + type DesktopRegionScreenshotQuery, + type DesktopScreenshotQuery, + type DesktopStartRequest, + type DesktopStatusResponse, type FsActionResponse, type FsDeleteQuery, type FsEntriesQuery, @@ -1397,6 +1410,82 @@ export class SandboxAgent { return this.requestHealth(); } + async startDesktop(request: DesktopStartRequest = {}): Promise { + return this.requestJson("POST", `${API_PREFIX}/desktop/start`, { + body: request, + }); + } + + async stopDesktop(): Promise { + return this.requestJson("POST", `${API_PREFIX}/desktop/stop`); + } + + async getDesktopStatus(): Promise { + return this.requestJson("GET", `${API_PREFIX}/desktop/status`); + } + + async getDesktopDisplayInfo(): Promise { + return this.requestJson("GET", `${API_PREFIX}/desktop/display/info`); + } + + async takeDesktopScreenshot(query: DesktopScreenshotQuery = {}): Promise { + const response = await this.requestRaw("GET", `${API_PREFIX}/desktop/screenshot`, { + query, + accept: "image/png", + }); + const buffer = await response.arrayBuffer(); + return new Uint8Array(buffer); + } + + async takeDesktopRegionScreenshot(query: DesktopRegionScreenshotQuery): Promise { + const response = await this.requestRaw("GET", `${API_PREFIX}/desktop/screenshot/region`, { + query, + accept: "image/png", + }); + const buffer = await response.arrayBuffer(); + return new Uint8Array(buffer); + } + + async getDesktopMousePosition(): Promise { + return this.requestJson("GET", `${API_PREFIX}/desktop/mouse/position`); + } + + async moveDesktopMouse(request: DesktopMouseMoveRequest): Promise { + return this.requestJson("POST", `${API_PREFIX}/desktop/mouse/move`, { + body: request, + }); + } + + async clickDesktop(request: DesktopMouseClickRequest): Promise { + return this.requestJson("POST", `${API_PREFIX}/desktop/mouse/click`, { + body: request, + }); + } + + async dragDesktopMouse(request: DesktopMouseDragRequest): Promise { + return this.requestJson("POST", `${API_PREFIX}/desktop/mouse/drag`, { + body: request, + }); + } + + async scrollDesktop(request: DesktopMouseScrollRequest): Promise { + return this.requestJson("POST", `${API_PREFIX}/desktop/mouse/scroll`, { + body: request, + }); + } + + async typeDesktopText(request: DesktopKeyboardTypeRequest): Promise { + return this.requestJson("POST", `${API_PREFIX}/desktop/keyboard/type`, { + body: request, + }); + } + + async pressDesktopKey(request: DesktopKeyboardPressRequest): Promise { + return this.requestJson("POST", `${API_PREFIX}/desktop/keyboard/press`, { + body: request, + }); + } + async listAgents(options?: AgentQueryOptions): Promise { return this.requestJson("GET", `${API_PREFIX}/agents`, { query: toAgentQuery(options), diff --git a/sdks/typescript/src/generated/openapi.ts b/sdks/typescript/src/generated/openapi.ts index bd3acf0..60f1234 100644 --- a/sdks/typescript/src/generated/openapi.ts +++ b/sdks/typescript/src/generated/openapi.ts @@ -31,6 +31,109 @@ export interface paths { put: operations["put_v1_config_skills"]; delete: operations["delete_v1_config_skills"]; }; + "/v1/desktop/display/info": { + /** + * Get desktop display information. + * @description Performs a health-gated display query against the managed desktop and + * returns the current display identifier and resolution. + */ + get: operations["get_v1_desktop_display_info"]; + }; + "/v1/desktop/keyboard/press": { + /** + * Press a desktop keyboard shortcut. + * @description Performs a health-gated `xdotool key` operation against the managed + * desktop. + */ + post: operations["post_v1_desktop_keyboard_press"]; + }; + "/v1/desktop/keyboard/type": { + /** + * Type desktop keyboard text. + * @description Performs a health-gated `xdotool type` operation against the managed + * desktop. + */ + post: operations["post_v1_desktop_keyboard_type"]; + }; + "/v1/desktop/mouse/click": { + /** + * Click on the desktop. + * @description Performs a health-gated pointer move and click against the managed desktop + * and returns the resulting mouse position. + */ + post: operations["post_v1_desktop_mouse_click"]; + }; + "/v1/desktop/mouse/drag": { + /** + * Drag the desktop mouse. + * @description Performs a health-gated drag gesture against the managed desktop and + * returns the resulting mouse position. + */ + post: operations["post_v1_desktop_mouse_drag"]; + }; + "/v1/desktop/mouse/move": { + /** + * Move the desktop mouse. + * @description Performs a health-gated absolute pointer move on the managed desktop and + * returns the resulting mouse position. + */ + post: operations["post_v1_desktop_mouse_move"]; + }; + "/v1/desktop/mouse/position": { + /** + * Get the current desktop mouse position. + * @description Performs a health-gated mouse position query against the managed desktop. + */ + get: operations["get_v1_desktop_mouse_position"]; + }; + "/v1/desktop/mouse/scroll": { + /** + * Scroll the desktop mouse wheel. + * @description Performs a health-gated scroll gesture at the requested coordinates and + * returns the resulting mouse position. + */ + post: operations["post_v1_desktop_mouse_scroll"]; + }; + "/v1/desktop/screenshot": { + /** + * Capture a full desktop screenshot. + * @description Performs a health-gated full-frame screenshot of the managed desktop and + * returns PNG bytes. + */ + get: operations["get_v1_desktop_screenshot"]; + }; + "/v1/desktop/screenshot/region": { + /** + * Capture a desktop screenshot region. + * @description Performs a health-gated screenshot crop against the managed desktop and + * returns the requested PNG region bytes. + */ + get: operations["get_v1_desktop_screenshot_region"]; + }; + "/v1/desktop/start": { + /** + * Start the private desktop runtime. + * @description Lazily launches the managed Xvfb/openbox stack, validates display health, + * and returns the resulting desktop status snapshot. + */ + post: operations["post_v1_desktop_start"]; + }; + "/v1/desktop/status": { + /** + * Get desktop runtime status. + * @description Returns the current desktop runtime state, dependency status, active + * display metadata, and supervised process information. + */ + get: operations["get_v1_desktop_status"]; + }; + "/v1/desktop/stop": { + /** + * Stop the private desktop runtime. + * @description Terminates the managed openbox/Xvfb/dbus processes owned by the desktop + * runtime and returns the resulting status snapshot. + */ + post: operations["post_v1_desktop_stop"]; + }; "/v1/fs/entries": { get: operations["get_v1_fs_entries"]; }; @@ -233,6 +336,119 @@ export interface components { AgentListResponse: { agents: components["schemas"]["AgentInfo"][]; }; + DesktopActionResponse: { + ok: boolean; + }; + DesktopDisplayInfoResponse: { + display: string; + resolution: components["schemas"]["DesktopResolution"]; + }; + DesktopErrorInfo: { + code: string; + message: string; + }; + DesktopKeyboardPressRequest: { + key: string; + }; + DesktopKeyboardTypeRequest: { + /** Format: int32 */ + delayMs?: number | null; + text: string; + }; + /** @enum {string} */ + DesktopMouseButton: "left" | "middle" | "right"; + DesktopMouseClickRequest: { + button?: components["schemas"]["DesktopMouseButton"] | null; + /** Format: int32 */ + clickCount?: number | null; + /** Format: int32 */ + x: number; + /** Format: int32 */ + y: number; + }; + DesktopMouseDragRequest: { + button?: components["schemas"]["DesktopMouseButton"] | null; + /** Format: int32 */ + endX: number; + /** Format: int32 */ + endY: number; + /** Format: int32 */ + startX: number; + /** Format: int32 */ + startY: number; + }; + DesktopMouseMoveRequest: { + /** Format: int32 */ + x: number; + /** Format: int32 */ + y: number; + }; + DesktopMousePositionResponse: { + /** Format: int32 */ + screen?: number | null; + window?: string | null; + /** Format: int32 */ + x: number; + /** Format: int32 */ + y: number; + }; + DesktopMouseScrollRequest: { + /** Format: int32 */ + deltaX?: number | null; + /** Format: int32 */ + deltaY?: number | null; + /** Format: int32 */ + x: number; + /** Format: int32 */ + y: number; + }; + DesktopProcessInfo: { + logPath?: string | null; + name: string; + /** Format: int32 */ + pid?: number | null; + running: boolean; + }; + DesktopRegionScreenshotQuery: { + /** Format: int32 */ + height: number; + /** Format: int32 */ + width: number; + /** Format: int32 */ + x: number; + /** Format: int32 */ + y: number; + }; + DesktopResolution: { + /** Format: int32 */ + dpi?: number | null; + /** Format: int32 */ + height: number; + /** Format: int32 */ + width: number; + }; + DesktopScreenshotQuery: Record; + DesktopStartRequest: { + /** Format: int32 */ + dpi?: number | null; + /** Format: int32 */ + height?: number | null; + /** Format: int32 */ + width?: number | null; + }; + /** @enum {string} */ + DesktopState: "inactive" | "install_required" | "starting" | "active" | "stopping" | "failed"; + DesktopStatusResponse: { + display?: string | null; + installCommand?: string | null; + lastError?: components["schemas"]["DesktopErrorInfo"] | null; + missingDependencies?: string[]; + processes?: components["schemas"]["DesktopProcessInfo"][]; + resolution?: components["schemas"]["DesktopResolution"] | null; + runtimeLogPath?: string | null; + startedAt?: string | null; + state: components["schemas"]["DesktopState"]; + }; /** @enum {string} */ ErrorType: | "invalid_request" @@ -827,6 +1043,441 @@ export interface operations { }; }; }; + /** + * Get desktop display information. + * @description Performs a health-gated display query against the managed desktop and + * returns the current display identifier and resolution. + */ + get_v1_desktop_display_info: { + responses: { + /** @description Desktop display information */ + 200: { + content: { + "application/json": components["schemas"]["DesktopDisplayInfoResponse"]; + }; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or display query failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Press a desktop keyboard shortcut. + * @description Performs a health-gated `xdotool key` operation against the managed + * desktop. + */ + post_v1_desktop_keyboard_press: { + requestBody: { + content: { + "application/json": components["schemas"]["DesktopKeyboardPressRequest"]; + }; + }; + responses: { + /** @description Desktop keyboard action result */ + 200: { + content: { + "application/json": components["schemas"]["DesktopActionResponse"]; + }; + }; + /** @description Invalid keyboard press request */ + 400: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or input failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Type desktop keyboard text. + * @description Performs a health-gated `xdotool type` operation against the managed + * desktop. + */ + post_v1_desktop_keyboard_type: { + requestBody: { + content: { + "application/json": components["schemas"]["DesktopKeyboardTypeRequest"]; + }; + }; + responses: { + /** @description Desktop keyboard action result */ + 200: { + content: { + "application/json": components["schemas"]["DesktopActionResponse"]; + }; + }; + /** @description Invalid keyboard type request */ + 400: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or input failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Click on the desktop. + * @description Performs a health-gated pointer move and click against the managed desktop + * and returns the resulting mouse position. + */ + post_v1_desktop_mouse_click: { + requestBody: { + content: { + "application/json": components["schemas"]["DesktopMouseClickRequest"]; + }; + }; + responses: { + /** @description Desktop mouse position after click */ + 200: { + content: { + "application/json": components["schemas"]["DesktopMousePositionResponse"]; + }; + }; + /** @description Invalid mouse click request */ + 400: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or input failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Drag the desktop mouse. + * @description Performs a health-gated drag gesture against the managed desktop and + * returns the resulting mouse position. + */ + post_v1_desktop_mouse_drag: { + requestBody: { + content: { + "application/json": components["schemas"]["DesktopMouseDragRequest"]; + }; + }; + responses: { + /** @description Desktop mouse position after drag */ + 200: { + content: { + "application/json": components["schemas"]["DesktopMousePositionResponse"]; + }; + }; + /** @description Invalid mouse drag request */ + 400: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or input failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Move the desktop mouse. + * @description Performs a health-gated absolute pointer move on the managed desktop and + * returns the resulting mouse position. + */ + post_v1_desktop_mouse_move: { + requestBody: { + content: { + "application/json": components["schemas"]["DesktopMouseMoveRequest"]; + }; + }; + responses: { + /** @description Desktop mouse position after move */ + 200: { + content: { + "application/json": components["schemas"]["DesktopMousePositionResponse"]; + }; + }; + /** @description Invalid mouse move request */ + 400: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or input failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Get the current desktop mouse position. + * @description Performs a health-gated mouse position query against the managed desktop. + */ + get_v1_desktop_mouse_position: { + responses: { + /** @description Desktop mouse position */ + 200: { + content: { + "application/json": components["schemas"]["DesktopMousePositionResponse"]; + }; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or input check failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Scroll the desktop mouse wheel. + * @description Performs a health-gated scroll gesture at the requested coordinates and + * returns the resulting mouse position. + */ + post_v1_desktop_mouse_scroll: { + requestBody: { + content: { + "application/json": components["schemas"]["DesktopMouseScrollRequest"]; + }; + }; + responses: { + /** @description Desktop mouse position after scroll */ + 200: { + content: { + "application/json": components["schemas"]["DesktopMousePositionResponse"]; + }; + }; + /** @description Invalid mouse scroll request */ + 400: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or input failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Capture a full desktop screenshot. + * @description Performs a health-gated full-frame screenshot of the managed desktop and + * returns PNG bytes. + */ + get_v1_desktop_screenshot: { + responses: { + /** @description Desktop screenshot as PNG bytes */ + 200: { + content: never; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or screenshot capture failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Capture a desktop screenshot region. + * @description Performs a health-gated screenshot crop against the managed desktop and + * returns the requested PNG region bytes. + */ + get_v1_desktop_screenshot_region: { + parameters: { + query: { + /** @description Region x coordinate */ + x: number; + /** @description Region y coordinate */ + y: number; + /** @description Region width */ + width: number; + /** @description Region height */ + height: number; + }; + }; + responses: { + /** @description Desktop screenshot region as PNG bytes */ + 200: { + content: never; + }; + /** @description Invalid screenshot region */ + 400: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime is not ready */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime health or screenshot capture failed */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Start the private desktop runtime. + * @description Lazily launches the managed Xvfb/openbox stack, validates display health, + * and returns the resulting desktop status snapshot. + */ + post_v1_desktop_start: { + requestBody: { + content: { + "application/json": components["schemas"]["DesktopStartRequest"]; + }; + }; + responses: { + /** @description Desktop runtime status after start */ + 200: { + content: { + "application/json": components["schemas"]["DesktopStatusResponse"]; + }; + }; + /** @description Invalid desktop start request */ + 400: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime is already transitioning */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop API unsupported on this platform */ + 501: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + /** @description Desktop runtime could not be started */ + 503: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Get desktop runtime status. + * @description Returns the current desktop runtime state, dependency status, active + * display metadata, and supervised process information. + */ + get_v1_desktop_status: { + responses: { + /** @description Desktop runtime status */ + 200: { + content: { + "application/json": components["schemas"]["DesktopStatusResponse"]; + }; + }; + /** @description Authentication required */ + 401: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; + /** + * Stop the private desktop runtime. + * @description Terminates the managed openbox/Xvfb/dbus processes owned by the desktop + * runtime and returns the resulting status snapshot. + */ + post_v1_desktop_stop: { + responses: { + /** @description Desktop runtime status after stop */ + 200: { + content: { + "application/json": components["schemas"]["DesktopStatusResponse"]; + }; + }; + /** @description Desktop runtime is already transitioning */ + 409: { + content: { + "application/json": components["schemas"]["ProblemDetails"]; + }; + }; + }; + }; get_v1_fs_entries: { parameters: { query?: { diff --git a/sdks/typescript/src/index.ts b/sdks/typescript/src/index.ts index 99bc1b6..fdac2f1 100644 --- a/sdks/typescript/src/index.ts +++ b/sdks/typescript/src/index.ts @@ -48,6 +48,24 @@ export type { AgentInstallRequest, AgentInstallResponse, AgentListResponse, + DesktopActionResponse, + DesktopDisplayInfoResponse, + DesktopErrorInfo, + DesktopKeyboardPressRequest, + DesktopKeyboardTypeRequest, + DesktopMouseButton, + DesktopMouseClickRequest, + DesktopMouseDragRequest, + DesktopMouseMoveRequest, + DesktopMousePositionResponse, + DesktopMouseScrollRequest, + DesktopProcessInfo, + DesktopRegionScreenshotQuery, + DesktopResolution, + DesktopScreenshotQuery, + DesktopStartRequest, + DesktopState, + DesktopStatusResponse, FsActionResponse, FsDeleteQuery, FsEntriesQuery, diff --git a/sdks/typescript/src/types.ts b/sdks/typescript/src/types.ts index 6865690..782df27 100644 --- a/sdks/typescript/src/types.ts +++ b/sdks/typescript/src/types.ts @@ -4,6 +4,27 @@ import type { components, operations } from "./generated/openapi.ts"; export type ProblemDetails = components["schemas"]["ProblemDetails"]; export type HealthResponse = JsonResponse; +export type DesktopState = components["schemas"]["DesktopState"]; +export type DesktopResolution = components["schemas"]["DesktopResolution"]; +export type DesktopErrorInfo = components["schemas"]["DesktopErrorInfo"]; +export type DesktopProcessInfo = components["schemas"]["DesktopProcessInfo"]; +export type DesktopStatusResponse = JsonResponse; +export type DesktopStartRequest = JsonRequestBody; +export type DesktopScreenshotQuery = + QueryParams extends never + ? Record + : QueryParams; +export type DesktopRegionScreenshotQuery = QueryParams; +export type DesktopMousePositionResponse = JsonResponse; +export type DesktopMouseButton = components["schemas"]["DesktopMouseButton"]; +export type DesktopMouseMoveRequest = JsonRequestBody; +export type DesktopMouseClickRequest = JsonRequestBody; +export type DesktopMouseDragRequest = JsonRequestBody; +export type DesktopMouseScrollRequest = JsonRequestBody; +export type DesktopKeyboardTypeRequest = JsonRequestBody; +export type DesktopKeyboardPressRequest = JsonRequestBody; +export type DesktopActionResponse = JsonResponse; +export type DesktopDisplayInfoResponse = JsonResponse; export type AgentListResponse = JsonResponse; export type AgentInfo = components["schemas"]["AgentInfo"]; export type AgentQuery = QueryParams; diff --git a/sdks/typescript/tests/integration.test.ts b/sdks/typescript/tests/integration.test.ts index 003b0dd..daa58e6 100644 --- a/sdks/typescript/tests/integration.test.ts +++ b/sdks/typescript/tests/integration.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { existsSync } from "node:fs"; -import { mkdtempSync, rmSync } from "node:fs"; +import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -179,21 +179,166 @@ function nodeCommand(source: string): { command: string; args: string[] } { }; } +function writeExecutable(path: string, source: string): void { + writeFileSync(path, source, "utf8"); + chmodSync(path, 0o755); +} + +function prepareFakeDesktopEnv(root: string): Record { + const binDir = join(root, "bin"); + const xdgStateHome = join(root, "xdg-state"); + const fakeStateDir = join(root, "fake-state"); + mkdirSync(binDir, { recursive: true }); + mkdirSync(xdgStateHome, { recursive: true }); + mkdirSync(fakeStateDir, { recursive: true }); + + writeExecutable( + join(binDir, "Xvfb"), + `#!/usr/bin/env sh +set -eu +display="\${1:-:191}" +number="\${display#:}" +socket="/tmp/.X11-unix/X\${number}" +mkdir -p /tmp/.X11-unix +touch "$socket" +cleanup() { + rm -f "$socket" + exit 0 +} +trap cleanup INT TERM EXIT +while :; do + sleep 1 +done +`, + ); + + writeExecutable( + join(binDir, "openbox"), + `#!/usr/bin/env sh +set -eu +trap 'exit 0' INT TERM +while :; do + sleep 1 +done +`, + ); + + writeExecutable( + join(binDir, "dbus-launch"), + `#!/usr/bin/env sh +set -eu +echo "DBUS_SESSION_BUS_ADDRESS=unix:path=/tmp/sandbox-agent-test-bus" +echo "DBUS_SESSION_BUS_PID=$$" +`, + ); + + writeExecutable( + join(binDir, "xrandr"), + `#!/usr/bin/env sh +set -eu +cat <<'EOF' +Screen 0: minimum 1 x 1, current 1440 x 900, maximum 32767 x 32767 +EOF +`, + ); + + writeExecutable( + join(binDir, "import"), + `#!/usr/bin/env sh +set -eu +printf '\\211PNG\\r\\n\\032\\n\\000\\000\\000\\rIHDR\\000\\000\\000\\001\\000\\000\\000\\001\\010\\006\\000\\000\\000\\037\\025\\304\\211\\000\\000\\000\\013IDATx\\234c\\000\\001\\000\\000\\005\\000\\001\\r\\n-\\264\\000\\000\\000\\000IEND\\256B\`\\202' +`, + ); + + writeExecutable( + join(binDir, "xdotool"), + `#!/usr/bin/env sh +set -eu +state_dir="\${SANDBOX_AGENT_DESKTOP_FAKE_STATE_DIR:?missing fake state dir}" +state_file="\${state_dir}/mouse" +mkdir -p "$state_dir" +if [ ! -f "$state_file" ]; then + printf '0 0\\n' > "$state_file" +fi + +read_state() { + read -r x y < "$state_file" +} + +write_state() { + printf '%s %s\\n' "$1" "$2" > "$state_file" +} + +command="\${1:-}" +case "$command" in + getmouselocation) + read_state + printf 'X=%s\\nY=%s\\nSCREEN=0\\nWINDOW=0\\n' "$x" "$y" + ;; + mousemove) + shift + x="\${1:-0}" + y="\${2:-0}" + shift 2 || true + while [ "$#" -gt 0 ]; do + token="$1" + shift + case "$token" in + mousemove) + x="\${1:-0}" + y="\${2:-0}" + shift 2 || true + ;; + mousedown|mouseup) + shift 1 || true + ;; + click) + if [ "\${1:-}" = "--repeat" ]; then + shift 2 || true + fi + shift 1 || true + ;; + esac + done + write_state "$x" "$y" + ;; + type|key) + exit 0 + ;; + *) + exit 0 + ;; +esac +`, + ); + + return { + SANDBOX_AGENT_DESKTOP_TEST_ASSUME_LINUX: "1", + SANDBOX_AGENT_DESKTOP_DISPLAY_NUM: "191", + SANDBOX_AGENT_DESKTOP_FAKE_STATE_DIR: fakeStateDir, + XDG_STATE_HOME: xdgStateHome, + PATH: `${binDir}:${process.env.PATH ?? ""}`, + }; +} + describe("Integration: TypeScript SDK flat session API", () => { let handle: SandboxAgentSpawnHandle; let baseUrl: string; let token: string; let dataHome: string; + let desktopHome: string; beforeAll(async () => { dataHome = mkdtempSync(join(tmpdir(), "sdk-integration-")); + desktopHome = mkdtempSync(join(tmpdir(), "sdk-desktop-")); const agentEnv = prepareMockAgentDataHome(dataHome); + const desktopEnv = prepareFakeDesktopEnv(desktopHome); handle = await spawnSandboxAgent({ enabled: true, log: "silent", timeoutMs: 30000, - env: agentEnv, + env: { ...agentEnv, ...desktopEnv }, }); baseUrl = handle.baseUrl; token = handle.token; @@ -202,6 +347,7 @@ describe("Integration: TypeScript SDK flat session API", () => { afterAll(async () => { await handle.dispose(); rmSync(dataHome, { recursive: true, force: true }); + rmSync(desktopHome, { recursive: true, force: true }); }); it("detects Node.js runtime", () => { @@ -957,4 +1103,91 @@ describe("Integration: TypeScript SDK flat session API", () => { await sdk.dispose(); } }); + + it("covers desktop status, screenshot, display, mouse, and keyboard helpers", async () => { + const sdk = await SandboxAgent.connect({ + baseUrl, + token, + }); + + try { + const initialStatus = await sdk.getDesktopStatus(); + expect(initialStatus.state).toBe("inactive"); + + const started = await sdk.startDesktop({ + width: 1440, + height: 900, + dpi: 96, + }); + expect(started.state).toBe("active"); + expect(started.display?.startsWith(":")).toBe(true); + expect(started.missingDependencies).toEqual([]); + + const displayInfo = await sdk.getDesktopDisplayInfo(); + expect(displayInfo.display).toBe(started.display); + expect(displayInfo.resolution.width).toBe(1440); + expect(displayInfo.resolution.height).toBe(900); + + const screenshot = await sdk.takeDesktopScreenshot(); + expect(Buffer.from(screenshot.subarray(0, 8)).equals(Buffer.from("\x89PNG\r\n\x1a\n", "binary"))).toBe(true); + + const region = await sdk.takeDesktopRegionScreenshot({ + x: 10, + y: 20, + width: 40, + height: 50, + }); + expect(Buffer.from(region.subarray(0, 8)).equals(Buffer.from("\x89PNG\r\n\x1a\n", "binary"))).toBe(true); + + const moved = await sdk.moveDesktopMouse({ x: 40, y: 50 }); + expect(moved.x).toBe(40); + expect(moved.y).toBe(50); + + const dragged = await sdk.dragDesktopMouse({ + startX: 40, + startY: 50, + endX: 80, + endY: 90, + button: "left", + }); + expect(dragged.x).toBe(80); + expect(dragged.y).toBe(90); + + const clicked = await sdk.clickDesktop({ + x: 80, + y: 90, + button: "left", + clickCount: 1, + }); + expect(clicked.x).toBe(80); + expect(clicked.y).toBe(90); + + const scrolled = await sdk.scrollDesktop({ + x: 80, + y: 90, + deltaY: -2, + }); + expect(scrolled.x).toBe(80); + expect(scrolled.y).toBe(90); + + const position = await sdk.getDesktopMousePosition(); + expect(position.x).toBe(80); + expect(position.y).toBe(90); + + const typed = await sdk.typeDesktopText({ + text: "hello desktop", + delayMs: 5, + }); + expect(typed.ok).toBe(true); + + const pressed = await sdk.pressDesktopKey({ key: "ctrl+l" }); + expect(pressed.ok).toBe(true); + + const stopped = await sdk.stopDesktop(); + expect(stopped.state).toBe("inactive"); + } finally { + await sdk.stopDesktop().catch(() => {}); + await sdk.dispose(); + } + }); }); diff --git a/server/packages/sandbox-agent/src/cli.rs b/server/packages/sandbox-agent/src/cli.rs index 1c12e4b..fa1005f 100644 --- a/server/packages/sandbox-agent/src/cli.rs +++ b/server/packages/sandbox-agent/src/cli.rs @@ -11,6 +11,9 @@ mod build_version { include!(concat!(env!("OUT_DIR"), "/version.rs")); } +use crate::desktop_install::{ + install_desktop, DesktopInstallRequest, DesktopPackageManager, +}; use crate::router::{ build_router_with_state, shutdown_servers, AppState, AuthConfig, BrandingMode, }; @@ -75,6 +78,8 @@ pub enum Command { Server(ServerArgs), /// Call the HTTP API without writing client code. Api(ApiArgs), + /// Install first-party runtime dependencies. + Install(InstallArgs), /// EXPERIMENTAL: OpenCode compatibility layer (disabled until ACP Phase 7). Opencode(OpencodeArgs), /// Manage the sandbox-agent background daemon. @@ -115,6 +120,12 @@ pub struct ApiArgs { command: ApiCommand, } +#[derive(Args, Debug)] +pub struct InstallArgs { + #[command(subcommand)] + command: InstallCommand, +} + #[derive(Args, Debug)] pub struct OpencodeArgs { #[arg(long, short = 'H', default_value = DEFAULT_HOST)] @@ -153,6 +164,12 @@ pub struct DaemonArgs { command: DaemonCommand, } +#[derive(Subcommand, Debug)] +pub enum InstallCommand { + /// Install desktop runtime dependencies. + Desktop(InstallDesktopArgs), +} + #[derive(Subcommand, Debug)] pub enum DaemonCommand { /// Start the daemon in the background. @@ -307,6 +324,18 @@ pub struct InstallAgentArgs { agent_process_version: Option, } +#[derive(Args, Debug)] +pub struct InstallDesktopArgs { + #[arg(long, default_value_t = false)] + yes: bool, + #[arg(long, default_value_t = false)] + print_only: bool, + #[arg(long, value_enum)] + package_manager: Option, + #[arg(long, default_value_t = false)] + no_fonts: bool, +} + #[derive(Args, Debug)] pub struct CredentialsExtractArgs { #[arg(long, short = 'a', value_enum)] @@ -402,6 +431,7 @@ pub fn run_command(command: &Command, cli: &CliConfig) -> Result<(), CliError> { match command { Command::Server(args) => run_server(cli, args), Command::Api(subcommand) => run_api(&subcommand.command, cli), + Command::Install(subcommand) => run_install(&subcommand.command), Command::Opencode(args) => run_opencode(cli, args), Command::Daemon(subcommand) => run_daemon(&subcommand.command, cli), Command::InstallAgent(args) => install_agent_local(args), @@ -409,6 +439,12 @@ pub fn run_command(command: &Command, cli: &CliConfig) -> Result<(), CliError> { } } +fn run_install(command: &InstallCommand) -> Result<(), CliError> { + match command { + InstallCommand::Desktop(args) => install_desktop_local(args), + } +} + fn run_server(cli: &CliConfig, server: &ServerArgs) -> Result<(), CliError> { let auth = if let Some(token) = cli.token.clone() { AuthConfig::with_token(token) @@ -473,6 +509,17 @@ fn run_api(command: &ApiCommand, cli: &CliConfig) -> Result<(), CliError> { } } +fn install_desktop_local(args: &InstallDesktopArgs) -> Result<(), CliError> { + install_desktop(DesktopInstallRequest { + yes: args.yes, + print_only: args.print_only, + package_manager: args.package_manager, + no_fonts: args.no_fonts, + }) + .map(|_| ()) + .map_err(CliError::Server) +} + fn run_agents(command: &AgentsCommand, cli: &CliConfig) -> Result<(), CliError> { match command { AgentsCommand::List(args) => { diff --git a/server/packages/sandbox-agent/src/desktop_errors.rs b/server/packages/sandbox-agent/src/desktop_errors.rs new file mode 100644 index 0000000..8d81c61 --- /dev/null +++ b/server/packages/sandbox-agent/src/desktop_errors.rs @@ -0,0 +1,158 @@ +use sandbox_agent_error::ProblemDetails; +use serde_json::{json, Map, Value}; + +use crate::desktop_types::{DesktopErrorInfo, DesktopProcessInfo}; + +#[derive(Debug, Clone)] +pub struct DesktopProblem { + status: u16, + title: &'static str, + code: &'static str, + message: String, + missing_dependencies: Vec, + install_command: Option, + processes: Vec, +} + +impl DesktopProblem { + pub fn unsupported_platform(message: impl Into) -> Self { + Self::new(501, "Desktop Unsupported", "desktop_unsupported_platform", message) + } + + pub fn dependencies_missing( + missing_dependencies: Vec, + install_command: Option, + processes: Vec, + ) -> Self { + let message = if missing_dependencies.is_empty() { + "Desktop dependencies are not installed".to_string() + } else { + format!( + "Desktop dependencies are not installed: {}", + missing_dependencies.join(", ") + ) + }; + Self::new( + 503, + "Desktop Dependencies Missing", + "desktop_dependencies_missing", + message, + ) + .with_missing_dependencies(missing_dependencies) + .with_install_command(install_command) + .with_processes(processes) + } + + pub fn runtime_inactive(message: impl Into) -> Self { + Self::new(409, "Desktop Runtime Inactive", "desktop_runtime_inactive", message) + } + + pub fn runtime_starting(message: impl Into) -> Self { + Self::new(409, "Desktop Runtime Starting", "desktop_runtime_starting", message) + } + + pub fn runtime_failed( + message: impl Into, + install_command: Option, + processes: Vec, + ) -> Self { + Self::new(503, "Desktop Runtime Failed", "desktop_runtime_failed", message) + .with_install_command(install_command) + .with_processes(processes) + } + + pub fn invalid_action(message: impl Into) -> Self { + Self::new(400, "Desktop Invalid Action", "desktop_invalid_action", message) + } + + pub fn screenshot_failed(message: impl Into, processes: Vec) -> Self { + Self::new(502, "Desktop Screenshot Failed", "desktop_screenshot_failed", message) + .with_processes(processes) + } + + pub fn input_failed(message: impl Into, processes: Vec) -> Self { + Self::new(502, "Desktop Input Failed", "desktop_input_failed", message) + .with_processes(processes) + } + + pub fn to_problem_details(&self) -> ProblemDetails { + let mut extensions = Map::new(); + extensions.insert("code".to_string(), Value::String(self.code.to_string())); + if !self.missing_dependencies.is_empty() { + extensions.insert( + "missingDependencies".to_string(), + Value::Array( + self.missing_dependencies + .iter() + .cloned() + .map(Value::String) + .collect(), + ), + ); + } + if let Some(install_command) = self.install_command.as_ref() { + extensions.insert( + "installCommand".to_string(), + Value::String(install_command.clone()), + ); + } + if !self.processes.is_empty() { + extensions.insert( + "processes".to_string(), + json!(self.processes), + ); + } + + ProblemDetails { + type_: format!("urn:sandbox-agent:error:{}", self.code), + title: self.title.to_string(), + status: self.status, + detail: Some(self.message.clone()), + instance: None, + extensions, + } + } + + pub fn to_error_info(&self) -> DesktopErrorInfo { + DesktopErrorInfo { + code: self.code.to_string(), + message: self.message.clone(), + } + } + + pub fn code(&self) -> &'static str { + self.code + } + + fn new( + status: u16, + title: &'static str, + code: &'static str, + message: impl Into, + ) -> Self { + Self { + status, + title, + code, + message: message.into(), + missing_dependencies: Vec::new(), + install_command: None, + processes: Vec::new(), + } + } + + fn with_missing_dependencies(mut self, missing_dependencies: Vec) -> Self { + self.missing_dependencies = missing_dependencies; + self + } + + fn with_install_command(mut self, install_command: Option) -> Self { + self.install_command = install_command; + self + } + + fn with_processes(mut self, processes: Vec) -> Self { + self.processes = processes; + self + } +} diff --git a/server/packages/sandbox-agent/src/desktop_install.rs b/server/packages/sandbox-agent/src/desktop_install.rs new file mode 100644 index 0000000..fa5766b --- /dev/null +++ b/server/packages/sandbox-agent/src/desktop_install.rs @@ -0,0 +1,282 @@ +use std::fmt; +use std::io::{self, Write}; +use std::path::PathBuf; +use std::process::Command as ProcessCommand; + +use clap::ValueEnum; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +pub enum DesktopPackageManager { + Apt, + Dnf, + Apk, +} + +#[derive(Debug, Clone)] +pub struct DesktopInstallRequest { + pub yes: bool, + pub print_only: bool, + pub package_manager: Option, + pub no_fonts: bool, +} + +pub fn install_desktop(request: DesktopInstallRequest) -> Result<(), String> { + if std::env::consts::OS != "linux" { + return Err("desktop installation is only supported on Linux hosts and sandboxes".to_string()); + } + + let package_manager = match request.package_manager { + Some(value) => value, + None => detect_package_manager().ok_or_else(|| { + "could not detect a supported package manager (expected apt, dnf, or apk)".to_string() + })?, + }; + + let packages = desktop_packages(package_manager, request.no_fonts); + let used_sudo = !running_as_root() && find_binary("sudo").is_some(); + if !running_as_root() && !used_sudo { + return Err( + "desktop installation requires root or sudo access; rerun as root or install dependencies manually" + .to_string(), + ); + } + + println!("Desktop package manager: {}", package_manager); + println!("Desktop packages:"); + for package in &packages { + println!(" - {package}"); + } + println!("Install command:"); + println!(" {}", render_install_command(package_manager, used_sudo, &packages)); + + if request.print_only { + return Ok(()); + } + + if !request.yes && !prompt_yes_no("Proceed with desktop dependency installation? [y/N] ")? { + return Err("installation cancelled".to_string()); + } + + run_install_commands(package_manager, used_sudo, &packages)?; + + println!("Desktop dependencies installed."); + Ok(()) +} + +fn detect_package_manager() -> Option { + if find_binary("apt-get").is_some() { + return Some(DesktopPackageManager::Apt); + } + if find_binary("dnf").is_some() { + return Some(DesktopPackageManager::Dnf); + } + if find_binary("apk").is_some() { + return Some(DesktopPackageManager::Apk); + } + None +} + +fn desktop_packages( + package_manager: DesktopPackageManager, + no_fonts: bool, +) -> Vec { + let mut packages = match package_manager { + DesktopPackageManager::Apt => vec![ + "xvfb", + "openbox", + "xdotool", + "imagemagick", + "x11-xserver-utils", + "dbus-x11", + "xauth", + "fonts-dejavu-core", + ], + DesktopPackageManager::Dnf => vec![ + "xorg-x11-server-Xvfb", + "openbox", + "xdotool", + "ImageMagick", + "xrandr", + "dbus-x11", + "xauth", + "dejavu-sans-fonts", + ], + DesktopPackageManager::Apk => vec![ + "xvfb", + "openbox", + "xdotool", + "imagemagick", + "xrandr", + "dbus", + "xauth", + "ttf-dejavu", + ], + } + .into_iter() + .map(str::to_string) + .collect::>(); + + if no_fonts { + packages.retain(|package| { + package != "fonts-dejavu-core" + && package != "dejavu-sans-fonts" + && package != "ttf-dejavu" + }); + } + + packages +} + +fn render_install_command( + package_manager: DesktopPackageManager, + used_sudo: bool, + packages: &[String], +) -> String { + let sudo = if used_sudo { "sudo " } else { "" }; + match package_manager { + DesktopPackageManager::Apt => format!( + "{sudo}apt-get update && {sudo}env DEBIAN_FRONTEND=noninteractive apt-get install -y {}", + packages.join(" ") + ), + DesktopPackageManager::Dnf => { + format!("{sudo}dnf install -y {}", packages.join(" ")) + } + DesktopPackageManager::Apk => { + format!("{sudo}apk add --no-cache {}", packages.join(" ")) + } + } +} + +fn run_install_commands( + package_manager: DesktopPackageManager, + used_sudo: bool, + packages: &[String], +) -> Result<(), String> { + match package_manager { + DesktopPackageManager::Apt => { + run_command(command_with_privilege( + used_sudo, + "apt-get", + vec!["update".to_string()], + ))?; + let mut args = vec![ + "DEBIAN_FRONTEND=noninteractive".to_string(), + "apt-get".to_string(), + "install".to_string(), + "-y".to_string(), + ]; + args.extend(packages.iter().cloned()); + run_command(command_with_privilege(used_sudo, "env", args))?; + } + DesktopPackageManager::Dnf => { + let mut args = vec!["install".to_string(), "-y".to_string()]; + args.extend(packages.iter().cloned()); + run_command(command_with_privilege(used_sudo, "dnf", args))?; + } + DesktopPackageManager::Apk => { + let mut args = vec!["add".to_string(), "--no-cache".to_string()]; + args.extend(packages.iter().cloned()); + run_command(command_with_privilege(used_sudo, "apk", args))?; + } + } + Ok(()) +} + +fn command_with_privilege( + used_sudo: bool, + program: &str, + args: Vec, +) -> (String, Vec) { + if used_sudo { + let mut sudo_args = vec![program.to_string()]; + sudo_args.extend(args); + ("sudo".to_string(), sudo_args) + } else { + (program.to_string(), args) + } +} + +fn run_command((program, args): (String, Vec)) -> Result<(), String> { + let status = ProcessCommand::new(&program) + .args(&args) + .status() + .map_err(|err| format!("failed to run `{program}`: {err}"))?; + if !status.success() { + return Err(format!( + "command `{}` exited with status {}", + format_command(&program, &args), + status + )); + } + Ok(()) +} + +fn prompt_yes_no(prompt: &str) -> Result { + print!("{prompt}"); + io::stdout() + .flush() + .map_err(|err| format!("failed to flush prompt: {err}"))?; + let mut input = String::new(); + io::stdin() + .read_line(&mut input) + .map_err(|err| format!("failed to read confirmation: {err}"))?; + let normalized = input.trim().to_ascii_lowercase(); + Ok(matches!(normalized.as_str(), "y" | "yes")) +} + +fn running_as_root() -> bool { + #[cfg(unix)] + unsafe { + return libc::geteuid() == 0; + } + #[cfg(not(unix))] + { + false + } +} + +fn find_binary(name: &str) -> Option { + let path_env = std::env::var_os("PATH")?; + for path in std::env::split_paths(&path_env) { + let candidate = path.join(name); + if candidate.is_file() { + return Some(candidate); + } + } + None +} + +fn format_command(program: &str, args: &[String]) -> String { + let mut parts = vec![program.to_string()]; + parts.extend(args.iter().cloned()); + parts.join(" ") +} + +impl fmt::Display for DesktopPackageManager { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + DesktopPackageManager::Apt => write!(f, "apt"), + DesktopPackageManager::Dnf => write!(f, "dnf"), + DesktopPackageManager::Apk => write!(f, "apk"), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn desktop_packages_support_no_fonts() { + let packages = desktop_packages(DesktopPackageManager::Apt, true); + assert!(!packages.iter().any(|value| value == "fonts-dejavu-core")); + assert!(packages.iter().any(|value| value == "xvfb")); + } + + #[test] + fn render_install_command_matches_package_manager() { + let packages = vec!["xvfb".to_string(), "openbox".to_string()]; + let command = render_install_command(DesktopPackageManager::Apk, false, &packages); + assert_eq!(command, "apk add --no-cache xvfb openbox"); + } +} diff --git a/server/packages/sandbox-agent/src/desktop_runtime.rs b/server/packages/sandbox-agent/src/desktop_runtime.rs new file mode 100644 index 0000000..09ff55a --- /dev/null +++ b/server/packages/sandbox-agent/src/desktop_runtime.rs @@ -0,0 +1,1313 @@ +use std::collections::HashMap; +use std::fs::{self, OpenOptions}; +use std::path::{Path, PathBuf}; +use std::process::{Output, Stdio}; +use std::sync::Arc; +use std::time::Duration; + +use tokio::process::{Child, Command}; +use tokio::sync::Mutex; + +use crate::desktop_errors::DesktopProblem; +use crate::desktop_types::{ + DesktopActionResponse, DesktopDisplayInfoResponse, DesktopErrorInfo, DesktopKeyboardPressRequest, + DesktopKeyboardTypeRequest, DesktopMouseButton, DesktopMouseClickRequest, + DesktopMouseDragRequest, DesktopMouseMoveRequest, DesktopMousePositionResponse, + DesktopMouseScrollRequest, DesktopProcessInfo, DesktopRegionScreenshotQuery, DesktopResolution, + DesktopStartRequest, DesktopState, DesktopStatusResponse, +}; + +const DEFAULT_WIDTH: u32 = 1440; +const DEFAULT_HEIGHT: u32 = 900; +const DEFAULT_DPI: u32 = 96; +const DEFAULT_DISPLAY_NUM: i32 = 99; +const MAX_DISPLAY_PROBE: i32 = 10; +const SCREENSHOT_TIMEOUT: Duration = Duration::from_secs(10); +const INPUT_TIMEOUT: Duration = Duration::from_secs(5); +const STARTUP_TIMEOUT: Duration = Duration::from_secs(15); +const PNG_SIGNATURE: &[u8] = b"\x89PNG\r\n\x1a\n"; + +#[derive(Debug, Clone)] +pub struct DesktopRuntime { + config: DesktopRuntimeConfig, + inner: Arc>, +} + +#[derive(Debug, Clone)] +pub struct DesktopRuntimeConfig { + state_dir: PathBuf, + display_num: i32, + assume_linux_for_tests: bool, +} + +#[derive(Debug)] +struct DesktopRuntimeStateData { + state: DesktopState, + display_num: i32, + display: Option, + resolution: Option, + started_at: Option, + last_error: Option, + missing_dependencies: Vec, + install_command: Option, + runtime_log_path: PathBuf, + environment: HashMap, + xvfb: Option, + openbox: Option, + dbus_pid: Option, +} + +#[derive(Debug)] +struct ManagedDesktopChild { + name: &'static str, + child: Child, + log_path: PathBuf, +} + +#[derive(Debug, Clone)] +struct DesktopReadyContext { + display: String, + environment: HashMap, + resolution: DesktopResolution, +} + +impl Default for DesktopRuntimeConfig { + fn default() -> Self { + let display_num = std::env::var("SANDBOX_AGENT_DESKTOP_DISPLAY_NUM") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or(DEFAULT_DISPLAY_NUM); + + let state_dir = std::env::var("SANDBOX_AGENT_DESKTOP_STATE_DIR") + .ok() + .map(PathBuf::from) + .unwrap_or_else(default_state_dir); + + let assume_linux_for_tests = std::env::var("SANDBOX_AGENT_DESKTOP_TEST_ASSUME_LINUX") + .ok() + .map(|value| value == "1" || value.eq_ignore_ascii_case("true")) + .unwrap_or(false); + + Self { + state_dir, + display_num, + assume_linux_for_tests, + } + } +} + +impl DesktopRuntime { + pub fn new() -> Self { + Self::with_config(DesktopRuntimeConfig::default()) + } + + pub fn with_config(config: DesktopRuntimeConfig) -> Self { + let runtime_log_path = config.state_dir.join("desktop-runtime.log"); + Self { + inner: Arc::new(Mutex::new(DesktopRuntimeStateData { + state: DesktopState::Inactive, + display_num: config.display_num, + display: None, + resolution: None, + started_at: None, + last_error: None, + missing_dependencies: Vec::new(), + install_command: None, + runtime_log_path, + environment: HashMap::new(), + xvfb: None, + openbox: None, + dbus_pid: None, + })), + config, + } + } + + pub async fn status(&self) -> DesktopStatusResponse { + let mut state = self.inner.lock().await; + self.refresh_status_locked(&mut state).await; + self.snapshot_locked(&state) + } + + pub async fn start( + &self, + request: DesktopStartRequest, + ) -> Result { + let mut state = self.inner.lock().await; + + if !self.platform_supported() { + let problem = DesktopProblem::unsupported_platform( + "Desktop APIs are only supported on Linux hosts and sandboxes", + ); + self.record_problem_locked(&mut state, &problem); + state.state = DesktopState::Failed; + return Err(problem); + } + + if matches!(state.state, DesktopState::Starting | DesktopState::Stopping) { + return Err(DesktopProblem::runtime_starting( + "Desktop runtime is busy transitioning state", + )); + } + + self.refresh_status_locked(&mut state).await; + if state.state == DesktopState::Active { + return Ok(self.snapshot_locked(&state)); + } + + if !state.missing_dependencies.is_empty() { + return Err(DesktopProblem::dependencies_missing( + state.missing_dependencies.clone(), + state.install_command.clone(), + self.processes_locked(&state), + )); + } + + self.ensure_state_dir_locked(&state) + .map_err(|err| DesktopProblem::runtime_failed(err, None, self.processes_locked(&state)))?; + self.write_runtime_log_locked(&state, "starting desktop runtime"); + + let width = request.width.unwrap_or(DEFAULT_WIDTH); + let height = request.height.unwrap_or(DEFAULT_HEIGHT); + let dpi = request.dpi.unwrap_or(DEFAULT_DPI); + validate_start_request(width, height, dpi)?; + + let display_num = self.choose_display_num()?; + let display = format!(":{display_num}"); + let resolution = DesktopResolution { + width, + height, + dpi: Some(dpi), + }; + + state.state = DesktopState::Starting; + state.display_num = display_num; + state.display = Some(display.clone()); + state.resolution = Some(resolution.clone()); + state.started_at = None; + state.last_error = None; + state.environment = self.base_environment(&display)?; + state.install_command = None; + + self.start_dbus_locked(&mut state).await?; + self.start_xvfb_locked(&mut state, &resolution).await?; + self.wait_for_socket(display_num).await?; + self.start_openbox_locked(&mut state).await?; + + let ready = DesktopReadyContext { + display, + environment: state.environment.clone(), + resolution, + }; + + let display_info = self + .query_display_info_locked(&state, &ready) + .await + .map_err(|problem| { + self.record_problem_locked(&mut state, &problem); + state.state = DesktopState::Failed; + problem + })?; + state.resolution = Some(display_info.resolution.clone()); + + self.capture_screenshot_locked(&state, None).await.map_err(|problem| { + self.record_problem_locked(&mut state, &problem); + state.state = DesktopState::Failed; + problem + })?; + + state.state = DesktopState::Active; + state.started_at = Some(chrono::Utc::now().to_rfc3339()); + state.last_error = None; + self.write_runtime_log_locked( + &state, + &format!( + "desktop runtime active on {} ({}x{}, dpi {})", + display_info.display, + display_info.resolution.width, + display_info.resolution.height, + display_info.resolution.dpi.unwrap_or(DEFAULT_DPI) + ), + ); + + Ok(self.snapshot_locked(&state)) + } + + pub async fn stop(&self) -> Result { + let mut state = self.inner.lock().await; + if matches!(state.state, DesktopState::Starting | DesktopState::Stopping) { + return Err(DesktopProblem::runtime_starting( + "Desktop runtime is busy transitioning state", + )); + } + + state.state = DesktopState::Stopping; + self.write_runtime_log_locked(&state, "stopping desktop runtime"); + + self.stop_openbox_locked(&mut state).await; + self.stop_xvfb_locked(&mut state).await; + self.stop_dbus_locked(&mut state); + + state.state = DesktopState::Inactive; + state.display = None; + state.resolution = None; + state.started_at = None; + state.last_error = None; + state.missing_dependencies = self.detect_missing_dependencies(); + state.install_command = self.install_command_for(&state.missing_dependencies); + state.environment.clear(); + + Ok(self.snapshot_locked(&state)) + } + + pub async fn shutdown(&self) { + let _ = self.stop().await; + } + + pub async fn screenshot(&self) -> Result, DesktopProblem> { + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + self.capture_screenshot_locked(&state, Some(&ready)).await + } + + pub async fn screenshot_region( + &self, + query: DesktopRegionScreenshotQuery, + ) -> Result, DesktopProblem> { + validate_region(&query)?; + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + let crop = format!("{}x{}+{}+{}", query.width, query.height, query.x, query.y); + self.capture_screenshot_with_crop_locked(&state, &ready, &crop).await + } + + pub async fn mouse_position(&self) -> Result { + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + self.mouse_position_locked(&state, &ready).await + } + + pub async fn move_mouse( + &self, + request: DesktopMouseMoveRequest, + ) -> Result { + validate_coordinates(request.x, request.y)?; + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + let args = vec![ + "mousemove".to_string(), + request.x.to_string(), + request.y.to_string(), + ]; + self.run_input_command_locked(&state, &ready, args).await?; + self.mouse_position_locked(&state, &ready).await + } + + pub async fn click_mouse( + &self, + request: DesktopMouseClickRequest, + ) -> Result { + validate_coordinates(request.x, request.y)?; + let click_count = request.click_count.unwrap_or(1); + if click_count == 0 { + return Err(DesktopProblem::invalid_action( + "clickCount must be greater than 0", + )); + } + + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + let button = mouse_button_code(request.button.unwrap_or(DesktopMouseButton::Left)); + let mut args = vec![ + "mousemove".to_string(), + request.x.to_string(), + request.y.to_string(), + "click".to_string(), + ]; + if click_count > 1 { + args.push("--repeat".to_string()); + args.push(click_count.to_string()); + } + args.push(button.to_string()); + self.run_input_command_locked(&state, &ready, args).await?; + self.mouse_position_locked(&state, &ready).await + } + + pub async fn drag_mouse( + &self, + request: DesktopMouseDragRequest, + ) -> Result { + validate_coordinates(request.start_x, request.start_y)?; + validate_coordinates(request.end_x, request.end_y)?; + + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + let button = mouse_button_code(request.button.unwrap_or(DesktopMouseButton::Left)); + let args = vec![ + "mousemove".to_string(), + request.start_x.to_string(), + request.start_y.to_string(), + "mousedown".to_string(), + button.to_string(), + "mousemove".to_string(), + request.end_x.to_string(), + request.end_y.to_string(), + "mouseup".to_string(), + button.to_string(), + ]; + self.run_input_command_locked(&state, &ready, args).await?; + self.mouse_position_locked(&state, &ready).await + } + + pub async fn scroll_mouse( + &self, + request: DesktopMouseScrollRequest, + ) -> Result { + validate_coordinates(request.x, request.y)?; + let delta_x = request.delta_x.unwrap_or(0); + let delta_y = request.delta_y.unwrap_or(0); + if delta_x == 0 && delta_y == 0 { + return Err(DesktopProblem::invalid_action( + "deltaX or deltaY must be non-zero", + )); + } + + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + let mut args = vec![ + "mousemove".to_string(), + request.x.to_string(), + request.y.to_string(), + ]; + + append_scroll_clicks(&mut args, delta_y, 4, 5); + append_scroll_clicks(&mut args, delta_x, 6, 7); + + self.run_input_command_locked(&state, &ready, args).await?; + self.mouse_position_locked(&state, &ready).await + } + + pub async fn type_text( + &self, + request: DesktopKeyboardTypeRequest, + ) -> Result { + if request.text.is_empty() { + return Err(DesktopProblem::invalid_action( + "text must not be empty", + )); + } + + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + let args = vec![ + "type".to_string(), + "--delay".to_string(), + request.delay_ms.unwrap_or(10).to_string(), + request.text, + ]; + self.run_input_command_locked(&state, &ready, args).await?; + Ok(DesktopActionResponse { ok: true }) + } + + pub async fn press_key( + &self, + request: DesktopKeyboardPressRequest, + ) -> Result { + if request.key.trim().is_empty() { + return Err(DesktopProblem::invalid_action("key must not be empty")); + } + + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + let args = vec!["key".to_string(), request.key]; + self.run_input_command_locked(&state, &ready, args).await?; + Ok(DesktopActionResponse { ok: true }) + } + + pub async fn display_info(&self) -> Result { + let mut state = self.inner.lock().await; + let ready = self.ensure_ready_locked(&mut state).await?; + self.query_display_info_locked(&state, &ready).await + } + + async fn ensure_ready_locked( + &self, + state: &mut DesktopRuntimeStateData, + ) -> Result { + self.refresh_status_locked(state).await; + match state.state { + DesktopState::Active => { + let display = state.display.clone().ok_or_else(|| { + DesktopProblem::runtime_failed( + "Desktop runtime has no active display", + state.install_command.clone(), + self.processes_locked(state), + ) + })?; + let resolution = state.resolution.clone().ok_or_else(|| { + DesktopProblem::runtime_failed( + "Desktop runtime has no active resolution", + state.install_command.clone(), + self.processes_locked(state), + ) + })?; + Ok(DesktopReadyContext { + display, + environment: state.environment.clone(), + resolution, + }) + } + DesktopState::InstallRequired => Err(DesktopProblem::dependencies_missing( + state.missing_dependencies.clone(), + state.install_command.clone(), + self.processes_locked(state), + )), + DesktopState::Inactive => Err(DesktopProblem::runtime_inactive( + "Desktop runtime has not been started", + )), + DesktopState::Starting | DesktopState::Stopping => Err(DesktopProblem::runtime_starting( + "Desktop runtime is still transitioning", + )), + DesktopState::Failed => Err(DesktopProblem::runtime_failed( + state + .last_error + .as_ref() + .map(|error| error.message.clone()) + .unwrap_or_else(|| "Desktop runtime is unhealthy".to_string()), + state.install_command.clone(), + self.processes_locked(state), + )), + } + } + + async fn refresh_status_locked(&self, state: &mut DesktopRuntimeStateData) { + let missing_dependencies = if self.platform_supported() { + self.detect_missing_dependencies() + } else { + Vec::new() + }; + state.missing_dependencies = missing_dependencies.clone(); + state.install_command = self.install_command_for(&missing_dependencies); + + if !self.platform_supported() { + state.state = DesktopState::Failed; + state.last_error = Some( + DesktopProblem::unsupported_platform( + "Desktop APIs are only supported on Linux hosts and sandboxes", + ) + .to_error_info(), + ); + return; + } + + if !missing_dependencies.is_empty() { + state.state = DesktopState::InstallRequired; + state.last_error = Some( + DesktopProblem::dependencies_missing( + missing_dependencies, + state.install_command.clone(), + self.processes_locked(state), + ) + .to_error_info(), + ); + return; + } + + if matches!(state.state, DesktopState::Inactive | DesktopState::Starting | DesktopState::Stopping) { + if state.state == DesktopState::Inactive { + state.last_error = None; + } + return; + } + + let Some(display) = state.display.clone() else { + state.state = DesktopState::Failed; + state.last_error = Some( + DesktopProblem::runtime_failed( + "Desktop runtime has no display", + None, + self.processes_locked(state), + ) + .to_error_info(), + ); + return; + }; + + if let Err(problem) = self.ensure_process_running_locked(state, "Xvfb").await { + self.record_problem_locked(state, &problem); + state.state = DesktopState::Failed; + return; + } + if let Err(problem) = self.ensure_process_running_locked(state, "openbox").await { + self.record_problem_locked(state, &problem); + state.state = DesktopState::Failed; + return; + } + + if !socket_path(state.display_num).exists() { + let problem = DesktopProblem::runtime_failed( + format!("X socket for display {display} is missing"), + state.install_command.clone(), + self.processes_locked(state), + ); + self.record_problem_locked(state, &problem); + state.state = DesktopState::Failed; + return; + } + + let ready = DesktopReadyContext { + display, + environment: state.environment.clone(), + resolution: state.resolution.clone().unwrap_or(DesktopResolution { + width: DEFAULT_WIDTH, + height: DEFAULT_HEIGHT, + dpi: Some(DEFAULT_DPI), + }), + }; + + match self.query_display_info_locked(state, &ready).await { + Ok(display_info) => { + state.resolution = Some(display_info.resolution); + } + Err(problem) => { + self.record_problem_locked(state, &problem); + state.state = DesktopState::Failed; + return; + } + } + + if let Err(problem) = self.capture_screenshot_locked(state, Some(&ready)).await { + self.record_problem_locked(state, &problem); + state.state = DesktopState::Failed; + return; + } + + state.state = DesktopState::Active; + state.last_error = None; + } + + async fn ensure_process_running_locked( + &self, + state: &mut DesktopRuntimeStateData, + name: &str, + ) -> Result<(), DesktopProblem> { + let process = match name { + "Xvfb" => state.xvfb.as_mut(), + "openbox" => state.openbox.as_mut(), + _ => None, + }; + + let Some(process) = process else { + return Err(DesktopProblem::runtime_failed( + format!("{name} is not running"), + state.install_command.clone(), + self.processes_locked(state), + )); + }; + + match process.child.try_wait() { + Ok(None) => Ok(()), + Ok(Some(status)) => Err(DesktopProblem::runtime_failed( + format!("{name} exited with status {status}"), + state.install_command.clone(), + self.processes_locked(state), + )), + Err(err) => Err(DesktopProblem::runtime_failed( + format!("failed to inspect {name}: {err}"), + state.install_command.clone(), + self.processes_locked(state), + )), + } + } + + async fn start_dbus_locked( + &self, + state: &mut DesktopRuntimeStateData, + ) -> Result<(), DesktopProblem> { + if find_binary("dbus-launch").is_none() { + self.write_runtime_log_locked(state, "dbus-launch not found; continuing without D-Bus session"); + return Ok(()); + } + + let output = run_command_output( + "dbus-launch", + &[], + &state.environment, + INPUT_TIMEOUT, + ) + .await + .map_err(|err| { + DesktopProblem::runtime_failed( + format!("failed to launch dbus-launch: {err}"), + None, + self.processes_locked(state), + ) + })?; + + if !output.status.success() { + self.write_runtime_log_locked( + state, + &format!( + "dbus-launch failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ), + ); + return Ok(()); + } + + for line in String::from_utf8_lossy(&output.stdout).lines() { + if let Some((key, value)) = line.split_once('=') { + let cleaned = value.trim().trim_end_matches(';').to_string(); + if key == "DBUS_SESSION_BUS_ADDRESS" { + state.environment.insert(key.to_string(), cleaned); + } else if key == "DBUS_SESSION_BUS_PID" { + state.dbus_pid = cleaned.parse::().ok(); + } + } + } + + Ok(()) + } + + async fn start_xvfb_locked( + &self, + state: &mut DesktopRuntimeStateData, + resolution: &DesktopResolution, + ) -> Result<(), DesktopProblem> { + let Some(display) = state.display.clone() else { + return Err(DesktopProblem::runtime_failed( + "Desktop display was not configured before starting Xvfb", + None, + self.processes_locked(state), + )); + }; + let args = vec![ + display, + "-screen".to_string(), + "0".to_string(), + format!("{}x{}x24", resolution.width, resolution.height), + "-dpi".to_string(), + resolution.dpi.unwrap_or(DEFAULT_DPI).to_string(), + "-nolisten".to_string(), + "tcp".to_string(), + ]; + let log_path = self.config.state_dir.join("desktop-xvfb.log"); + let child = self.spawn_logged_process("Xvfb", "Xvfb", &args, &state.environment, &log_path)?; + state.xvfb = Some(child); + Ok(()) + } + + async fn start_openbox_locked( + &self, + state: &mut DesktopRuntimeStateData, + ) -> Result<(), DesktopProblem> { + let log_path = self.config.state_dir.join("desktop-openbox.log"); + let child = + self.spawn_logged_process("openbox", "openbox", &[], &state.environment, &log_path)?; + state.openbox = Some(child); + Ok(()) + } + + async fn stop_xvfb_locked(&self, state: &mut DesktopRuntimeStateData) { + if let Some(mut child) = state.xvfb.take() { + self.write_runtime_log_locked(state, "stopping Xvfb"); + let _ = terminate_child(&mut child.child).await; + } + } + + async fn stop_openbox_locked(&self, state: &mut DesktopRuntimeStateData) { + if let Some(mut child) = state.openbox.take() { + self.write_runtime_log_locked(state, "stopping openbox"); + let _ = terminate_child(&mut child.child).await; + } + } + + fn stop_dbus_locked(&self, state: &mut DesktopRuntimeStateData) { + if let Some(pid) = state.dbus_pid.take() { + #[cfg(unix)] + unsafe { + libc::kill(pid as i32, libc::SIGTERM); + } + } + } + + async fn capture_screenshot_locked( + &self, + state: &DesktopRuntimeStateData, + ready: Option<&DesktopReadyContext>, + ) -> Result, DesktopProblem> { + match ready { + Some(ready) => self + .capture_screenshot_with_crop_locked(state, ready, "") + .await, + None => { + let ready = DesktopReadyContext { + display: state.display.clone().unwrap_or_else(|| format!(":{}", state.display_num)), + environment: state.environment.clone(), + resolution: state.resolution.clone().unwrap_or(DesktopResolution { + width: DEFAULT_WIDTH, + height: DEFAULT_HEIGHT, + dpi: Some(DEFAULT_DPI), + }), + }; + self.capture_screenshot_with_crop_locked(state, &ready, "").await + } + } + } + + async fn capture_screenshot_with_crop_locked( + &self, + state: &DesktopRuntimeStateData, + ready: &DesktopReadyContext, + crop: &str, + ) -> Result, DesktopProblem> { + let mut args = vec!["-window".to_string(), "root".to_string()]; + if !crop.is_empty() { + args.push("-crop".to_string()); + args.push(crop.to_string()); + } + args.push("png:-".to_string()); + + let output = run_command_output("import", &args, &ready.environment, SCREENSHOT_TIMEOUT) + .await + .map_err(|err| { + DesktopProblem::screenshot_failed( + format!("failed to capture desktop screenshot: {err}"), + self.processes_locked(state), + ) + })?; + if !output.status.success() { + return Err(DesktopProblem::screenshot_failed( + format!( + "desktop screenshot command failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ), + self.processes_locked(state), + )); + } + validate_png(&output.stdout).map_err(|message| { + DesktopProblem::screenshot_failed(message, self.processes_locked(state)) + })?; + Ok(output.stdout) + } + + async fn mouse_position_locked( + &self, + state: &DesktopRuntimeStateData, + ready: &DesktopReadyContext, + ) -> Result { + let args = vec!["getmouselocation".to_string(), "--shell".to_string()]; + let output = run_command_output("xdotool", &args, &ready.environment, INPUT_TIMEOUT) + .await + .map_err(|err| { + DesktopProblem::input_failed( + format!("failed to query mouse position: {err}"), + self.processes_locked(state), + ) + })?; + if !output.status.success() { + return Err(DesktopProblem::input_failed( + format!( + "mouse position command failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ), + self.processes_locked(state), + )); + } + parse_mouse_position(&output.stdout).map_err(|message| { + DesktopProblem::input_failed(message, self.processes_locked(state)) + }) + } + + async fn run_input_command_locked( + &self, + state: &DesktopRuntimeStateData, + ready: &DesktopReadyContext, + args: Vec, + ) -> Result<(), DesktopProblem> { + let output = run_command_output("xdotool", &args, &ready.environment, INPUT_TIMEOUT) + .await + .map_err(|err| { + DesktopProblem::input_failed( + format!("failed to execute desktop input command: {err}"), + self.processes_locked(state), + ) + })?; + if !output.status.success() { + return Err(DesktopProblem::input_failed( + format!( + "desktop input command failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ), + self.processes_locked(state), + )); + } + Ok(()) + } + + async fn query_display_info_locked( + &self, + state: &DesktopRuntimeStateData, + ready: &DesktopReadyContext, + ) -> Result { + let args = vec!["--current".to_string()]; + let output = run_command_output("xrandr", &args, &ready.environment, INPUT_TIMEOUT) + .await + .map_err(|err| { + DesktopProblem::runtime_failed( + format!("failed to query display info: {err}"), + state.install_command.clone(), + self.processes_locked(state), + ) + })?; + if !output.status.success() { + return Err(DesktopProblem::runtime_failed( + format!( + "display query failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ), + state.install_command.clone(), + self.processes_locked(state), + )); + } + let resolution = parse_xrandr_resolution(&output.stdout).map_err(|message| { + DesktopProblem::runtime_failed( + message, + state.install_command.clone(), + self.processes_locked(state), + ) + })?; + Ok(DesktopDisplayInfoResponse { + display: ready.display.clone(), + resolution: DesktopResolution { + dpi: ready.resolution.dpi, + ..resolution + }, + }) + } + + fn detect_missing_dependencies(&self) -> Vec { + let mut missing = Vec::new(); + for (name, binary) in [ + ("Xvfb", "Xvfb"), + ("openbox", "openbox"), + ("xdotool", "xdotool"), + ("import", "import"), + ("xrandr", "xrandr"), + ] { + if find_binary(binary).is_none() { + missing.push(name.to_string()); + } + } + missing + } + + fn install_command_for(&self, missing_dependencies: &[String]) -> Option { + if !self.platform_supported() || missing_dependencies.is_empty() { + None + } else { + Some("sandbox-agent install desktop --yes".to_string()) + } + } + + fn platform_supported(&self) -> bool { + cfg!(target_os = "linux") || self.config.assume_linux_for_tests + } + + fn choose_display_num(&self) -> Result { + for offset in 0..MAX_DISPLAY_PROBE { + let candidate = self.config.display_num + offset; + if !socket_path(candidate).exists() { + return Ok(candidate); + } + } + Err(DesktopProblem::runtime_failed( + "unable to find an available X display starting at :99", + None, + Vec::new(), + )) + } + + fn base_environment(&self, display: &str) -> Result, DesktopProblem> { + let mut environment = HashMap::new(); + environment.insert("DISPLAY".to_string(), display.to_string()); + environment.insert("HOME".to_string(), self.config.state_dir.join("home").to_string_lossy().to_string()); + environment.insert( + "USER".to_string(), + std::env::var("USER").unwrap_or_else(|_| "sandbox-agent".to_string()), + ); + environment.insert( + "PATH".to_string(), + std::env::var("PATH").unwrap_or_default(), + ); + fs::create_dir_all(self.config.state_dir.join("home")).map_err(|err| { + DesktopProblem::runtime_failed(format!("failed to create desktop home: {err}"), None, Vec::new()) + })?; + Ok(environment) + } + + fn spawn_logged_process( + &self, + name: &'static str, + command: &str, + args: &[String], + environment: &HashMap, + log_path: &Path, + ) -> Result { + if let Some(parent) = log_path.parent() { + fs::create_dir_all(parent).map_err(|err| { + DesktopProblem::runtime_failed( + format!("failed to create desktop log directory: {err}"), + None, + Vec::new(), + ) + })?; + } + + let stdout = OpenOptions::new() + .create(true) + .append(true) + .open(log_path) + .map_err(|err| { + DesktopProblem::runtime_failed( + format!("failed to open desktop log file {}: {err}", log_path.display()), + None, + Vec::new(), + ) + })?; + let stderr = stdout.try_clone().map_err(|err| { + DesktopProblem::runtime_failed( + format!("failed to clone desktop log file {}: {err}", log_path.display()), + None, + Vec::new(), + ) + })?; + + let mut child = Command::new(command); + child.args(args); + child.envs(environment); + child.stdin(Stdio::null()); + child.stdout(Stdio::from(stdout)); + child.stderr(Stdio::from(stderr)); + + let child = child.spawn().map_err(|err| { + DesktopProblem::runtime_failed( + format!("failed to spawn {name}: {err}"), + None, + Vec::new(), + ) + })?; + + Ok(ManagedDesktopChild { + name, + child, + log_path: log_path.to_path_buf(), + }) + } + + async fn wait_for_socket(&self, display_num: i32) -> Result<(), DesktopProblem> { + let socket = socket_path(display_num); + let parent = socket.parent().map(Path::to_path_buf).unwrap_or_else(|| PathBuf::from("/tmp/.X11-unix")); + let _ = fs::create_dir_all(parent); + + let start = tokio::time::Instant::now(); + while start.elapsed() < STARTUP_TIMEOUT { + if socket.exists() { + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + + Err(DesktopProblem::runtime_failed( + format!("timed out waiting for X socket {}", socket.display()), + None, + Vec::new(), + )) + } + + fn snapshot_locked(&self, state: &DesktopRuntimeStateData) -> DesktopStatusResponse { + DesktopStatusResponse { + state: state.state, + display: state.display.clone(), + resolution: state.resolution.clone(), + started_at: state.started_at.clone(), + last_error: state.last_error.clone(), + missing_dependencies: state.missing_dependencies.clone(), + install_command: state.install_command.clone(), + processes: self.processes_locked(state), + runtime_log_path: Some(state.runtime_log_path.to_string_lossy().to_string()), + } + } + + fn processes_locked(&self, state: &DesktopRuntimeStateData) -> Vec { + let mut processes = Vec::new(); + if let Some(child) = state.xvfb.as_ref() { + processes.push(DesktopProcessInfo { + name: child.name.to_string(), + pid: child.child.id(), + running: child_is_running(&child.child), + log_path: Some(child.log_path.to_string_lossy().to_string()), + }); + } + if let Some(child) = state.openbox.as_ref() { + processes.push(DesktopProcessInfo { + name: child.name.to_string(), + pid: child.child.id(), + running: child_is_running(&child.child), + log_path: Some(child.log_path.to_string_lossy().to_string()), + }); + } + if let Some(pid) = state.dbus_pid { + processes.push(DesktopProcessInfo { + name: "dbus".to_string(), + pid: Some(pid), + running: process_exists(pid), + log_path: None, + }); + } + processes + } + + fn record_problem_locked(&self, state: &mut DesktopRuntimeStateData, problem: &DesktopProblem) { + state.last_error = Some(problem.to_error_info()); + self.write_runtime_log_locked( + state, + &format!("{}: {}", problem.code(), problem.to_error_info().message), + ); + } + + fn ensure_state_dir_locked(&self, state: &DesktopRuntimeStateData) -> Result<(), String> { + fs::create_dir_all(&self.config.state_dir) + .map_err(|err| format!("failed to create desktop state dir {}: {err}", self.config.state_dir.display()))?; + if let Some(parent) = state.runtime_log_path.parent() { + fs::create_dir_all(parent) + .map_err(|err| format!("failed to create runtime log dir {}: {err}", parent.display()))?; + } + Ok(()) + } + + fn write_runtime_log_locked(&self, state: &DesktopRuntimeStateData, message: &str) { + if let Some(parent) = state.runtime_log_path.parent() { + let _ = fs::create_dir_all(parent); + } + let line = format!("{} {}\n", chrono::Utc::now().to_rfc3339(), message); + let _ = OpenOptions::new() + .create(true) + .append(true) + .open(&state.runtime_log_path) + .and_then(|mut file| std::io::Write::write_all(&mut file, line.as_bytes())); + } +} + +fn default_state_dir() -> PathBuf { + if let Ok(value) = std::env::var("XDG_STATE_HOME") { + return PathBuf::from(value).join("sandbox-agent").join("desktop"); + } + if let Some(home) = dirs::home_dir() { + return home.join(".local").join("state").join("sandbox-agent").join("desktop"); + } + std::env::temp_dir().join("sandbox-agent-desktop") +} + +fn socket_path(display_num: i32) -> PathBuf { + PathBuf::from(format!("/tmp/.X11-unix/X{display_num}")) +} + +fn find_binary(name: &str) -> Option { + let path_env = std::env::var_os("PATH")?; + for path in std::env::split_paths(&path_env) { + let candidate = path.join(name); + if candidate.is_file() { + return Some(candidate); + } + } + None +} + +async fn run_command_output( + command: &str, + args: &[String], + environment: &HashMap, + timeout: Duration, +) -> Result { + let mut child = Command::new(command); + child.args(args); + child.envs(environment); + child.stdin(Stdio::null()); + child.stdout(Stdio::piped()); + child.stderr(Stdio::piped()); + + let output = tokio::time::timeout(timeout, child.output()) + .await + .map_err(|_| format!("command timed out after {}s", timeout.as_secs()))? + .map_err(|err| err.to_string())?; + Ok(output) +} + +async fn terminate_child(child: &mut Child) -> Result<(), String> { + if let Ok(Some(_)) = child.try_wait() { + return Ok(()); + } + child.start_kill().map_err(|err| err.to_string())?; + let _ = tokio::time::timeout(Duration::from_secs(5), child.wait()).await; + Ok(()) +} + +fn child_is_running(child: &Child) -> bool { + child.id().is_some() +} + +fn process_exists(pid: u32) -> bool { + #[cfg(unix)] + unsafe { + return libc::kill(pid as i32, 0) == 0 || std::io::Error::last_os_error().raw_os_error() != Some(libc::ESRCH); + } + #[cfg(not(unix))] + { + let _ = pid; + false + } +} + +fn validate_png(bytes: &[u8]) -> Result<(), String> { + if bytes.len() < PNG_SIGNATURE.len() || &bytes[..PNG_SIGNATURE.len()] != PNG_SIGNATURE { + return Err("desktop screenshot did not return PNG bytes".to_string()); + } + Ok(()) +} + +fn parse_xrandr_resolution(bytes: &[u8]) -> Result { + let text = String::from_utf8_lossy(bytes); + for line in text.lines() { + if let Some(index) = line.find(" current ") { + let tail = &line[index + " current ".len()..]; + let mut parts = tail.split(','); + if let Some(current) = parts.next() { + let dims: Vec<&str> = current.split_whitespace().collect(); + if dims.len() >= 3 { + let width = dims[0].parse::().map_err(|_| "failed to parse xrandr width".to_string())?; + let height = dims[2].parse::().map_err(|_| "failed to parse xrandr height".to_string())?; + return Ok(DesktopResolution { + width, + height, + dpi: None, + }); + } + } + } + } + Err("unable to parse xrandr current resolution".to_string()) +} + +fn parse_mouse_position(bytes: &[u8]) -> Result { + let text = String::from_utf8_lossy(bytes); + let mut x = None; + let mut y = None; + let mut screen = None; + let mut window = None; + for line in text.lines() { + if let Some((key, value)) = line.split_once('=') { + match key { + "X" => x = value.parse::().ok(), + "Y" => y = value.parse::().ok(), + "SCREEN" => screen = value.parse::().ok(), + "WINDOW" => window = Some(value.to_string()), + _ => {} + } + } + } + match (x, y) { + (Some(x), Some(y)) => Ok(DesktopMousePositionResponse { + x, + y, + screen, + window, + }), + _ => Err("unable to parse xdotool mouse position".to_string()), + } +} + +fn validate_start_request(width: u32, height: u32, dpi: u32) -> Result<(), DesktopProblem> { + if width == 0 || height == 0 { + return Err(DesktopProblem::invalid_action( + "Desktop width and height must be greater than 0", + )); + } + if dpi == 0 { + return Err(DesktopProblem::invalid_action( + "Desktop dpi must be greater than 0", + )); + } + Ok(()) +} + +fn validate_region(query: &DesktopRegionScreenshotQuery) -> Result<(), DesktopProblem> { + validate_coordinates(query.x, query.y)?; + if query.width == 0 || query.height == 0 { + return Err(DesktopProblem::invalid_action( + "Screenshot region width and height must be greater than 0", + )); + } + Ok(()) +} + +fn validate_coordinates(x: i32, y: i32) -> Result<(), DesktopProblem> { + if x < 0 || y < 0 { + return Err(DesktopProblem::invalid_action( + "Desktop coordinates must be non-negative", + )); + } + Ok(()) +} + +fn mouse_button_code(button: DesktopMouseButton) -> u8 { + match button { + DesktopMouseButton::Left => 1, + DesktopMouseButton::Middle => 2, + DesktopMouseButton::Right => 3, + } +} + +fn append_scroll_clicks(args: &mut Vec, delta: i32, up_button: u8, down_button: u8) { + if delta == 0 { + return; + } + let button = if delta > 0 { up_button } else { down_button }; + let repeat = delta.unsigned_abs(); + args.push("click".to_string()); + if repeat > 1 { + args.push("--repeat".to_string()); + args.push(repeat.to_string()); + } + args.push(button.to_string()); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_xrandr_resolution_reads_current_geometry() { + let bytes = b"Screen 0: minimum 1 x 1, current 1440 x 900, maximum 32767 x 32767\n"; + let parsed = parse_xrandr_resolution(bytes).expect("parse resolution"); + assert_eq!(parsed.width, 1440); + assert_eq!(parsed.height, 900); + } + + #[test] + fn parse_mouse_position_reads_shell_output() { + let bytes = b"X=123\nY=456\nSCREEN=0\nWINDOW=0\n"; + let parsed = parse_mouse_position(bytes).expect("parse mouse position"); + assert_eq!(parsed.x, 123); + assert_eq!(parsed.y, 456); + assert_eq!(parsed.screen, Some(0)); + assert_eq!(parsed.window.as_deref(), Some("0")); + } + + #[test] + fn png_validation_rejects_non_png_bytes() { + let error = validate_png(b"not png").expect_err("validation should fail"); + assert!(error.contains("PNG")); + } +} diff --git a/server/packages/sandbox-agent/src/desktop_types.rs b/server/packages/sandbox-agent/src/desktop_types.rs new file mode 100644 index 0000000..c6a7b7e --- /dev/null +++ b/server/packages/sandbox-agent/src/desktop_types.rs @@ -0,0 +1,173 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum DesktopState { + Inactive, + InstallRequired, + Starting, + Active, + Stopping, + Failed, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct DesktopResolution { + pub width: u32, + pub height: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dpi: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct DesktopErrorInfo { + pub code: String, + pub message: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct DesktopProcessInfo { + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pid: Option, + pub running: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub log_path: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct DesktopStatusResponse { + pub state: DesktopState, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub display: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resolution: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_error: Option, + #[serde(default)] + pub missing_dependencies: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub install_command: Option, + #[serde(default)] + pub processes: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub runtime_log_path: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema, Default)] +#[serde(rename_all = "camelCase")] +pub struct DesktopStartRequest { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub width: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub height: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dpi: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema, Default)] +#[serde(rename_all = "camelCase")] +pub struct DesktopScreenshotQuery {} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DesktopRegionScreenshotQuery { + pub x: i32, + pub y: i32, + pub width: u32, + pub height: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct DesktopMousePositionResponse { + pub x: i32, + pub y: i32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub screen: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub window: Option, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum DesktopMouseButton { + Left, + Middle, + Right, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DesktopMouseMoveRequest { + pub x: i32, + pub y: i32, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DesktopMouseClickRequest { + pub x: i32, + pub y: i32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub button: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub click_count: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DesktopMouseDragRequest { + pub start_x: i32, + pub start_y: i32, + pub end_x: i32, + pub end_y: i32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub button: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DesktopMouseScrollRequest { + pub x: i32, + pub y: i32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delta_x: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delta_y: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DesktopKeyboardTypeRequest { + pub text: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delay_ms: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DesktopKeyboardPressRequest { + pub key: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct DesktopActionResponse { + pub ok: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct DesktopDisplayInfoResponse { + pub display: String, + pub resolution: DesktopResolution, +} diff --git a/server/packages/sandbox-agent/src/lib.rs b/server/packages/sandbox-agent/src/lib.rs index e84b10b..e128b7c 100644 --- a/server/packages/sandbox-agent/src/lib.rs +++ b/server/packages/sandbox-agent/src/lib.rs @@ -1,6 +1,10 @@ //! Sandbox agent core utilities. mod acp_proxy_runtime; +mod desktop_install; +mod desktop_errors; +mod desktop_runtime; +pub mod desktop_types; pub mod cli; pub mod daemon; mod process_runtime; diff --git a/server/packages/sandbox-agent/src/router.rs b/server/packages/sandbox-agent/src/router.rs index 110c325..44db01f 100644 --- a/server/packages/sandbox-agent/src/router.rs +++ b/server/packages/sandbox-agent/src/router.rs @@ -37,6 +37,9 @@ use tracing::Span; use utoipa::{Modify, OpenApi, ToSchema}; use crate::acp_proxy_runtime::{AcpProxyRuntime, ProxyPostOutcome}; +use crate::desktop_errors::DesktopProblem; +use crate::desktop_runtime::DesktopRuntime; +use crate::desktop_types::*; use crate::process_runtime::{ decode_input_bytes, ProcessLogFilter, ProcessLogFilterStream, ProcessRuntime, ProcessRuntimeConfig, ProcessSnapshot, ProcessStartSpec, ProcessStatus, ProcessStream, RunSpec, @@ -87,6 +90,7 @@ pub struct AppState { acp_proxy: Arc, opencode_server_manager: Arc, process_runtime: Arc, + desktop_runtime: Arc, pub(crate) branding: BrandingMode, version_cache: Mutex>, } @@ -111,12 +115,14 @@ impl AppState { }, )); let process_runtime = Arc::new(ProcessRuntime::new()); + let desktop_runtime = Arc::new(DesktopRuntime::new()); Self { auth, agent_manager, acp_proxy, opencode_server_manager, process_runtime, + desktop_runtime, branding, version_cache: Mutex::new(HashMap::new()), } @@ -138,6 +144,10 @@ impl AppState { self.process_runtime.clone() } + pub(crate) fn desktop_runtime(&self) -> Arc { + self.desktop_runtime.clone() + } + pub(crate) fn purge_version_cache(&self, agent: AgentId) { self.version_cache.lock().unwrap().remove(&agent); } @@ -172,6 +182,22 @@ pub fn build_router(state: AppState) -> Router { pub fn build_router_with_state(shared: Arc) -> (Router, Arc) { let mut v1_router = Router::new() .route("/health", get(get_v1_health)) + .route("/desktop/status", get(get_v1_desktop_status)) + .route("/desktop/start", post(post_v1_desktop_start)) + .route("/desktop/stop", post(post_v1_desktop_stop)) + .route("/desktop/screenshot", get(get_v1_desktop_screenshot)) + .route( + "/desktop/screenshot/region", + get(get_v1_desktop_screenshot_region), + ) + .route("/desktop/mouse/position", get(get_v1_desktop_mouse_position)) + .route("/desktop/mouse/move", post(post_v1_desktop_mouse_move)) + .route("/desktop/mouse/click", post(post_v1_desktop_mouse_click)) + .route("/desktop/mouse/drag", post(post_v1_desktop_mouse_drag)) + .route("/desktop/mouse/scroll", post(post_v1_desktop_mouse_scroll)) + .route("/desktop/keyboard/type", post(post_v1_desktop_keyboard_type)) + .route("/desktop/keyboard/press", post(post_v1_desktop_keyboard_press)) + .route("/desktop/display/info", get(get_v1_desktop_display_info)) .route("/agents", get(get_v1_agents)) .route("/agents/:agent", get(get_v1_agent)) .route("/agents/:agent/install", post(post_v1_agent_install)) @@ -316,12 +342,26 @@ async fn opencode_unavailable() -> Response { pub async fn shutdown_servers(state: &Arc) { state.acp_proxy().shutdown_all().await; state.opencode_server_manager().shutdown().await; + state.desktop_runtime().shutdown().await; } #[derive(OpenApi)] #[openapi( paths( get_v1_health, + get_v1_desktop_status, + post_v1_desktop_start, + post_v1_desktop_stop, + get_v1_desktop_screenshot, + get_v1_desktop_screenshot_region, + get_v1_desktop_mouse_position, + post_v1_desktop_mouse_move, + post_v1_desktop_mouse_click, + post_v1_desktop_mouse_drag, + post_v1_desktop_mouse_scroll, + post_v1_desktop_keyboard_type, + post_v1_desktop_keyboard_press, + get_v1_desktop_display_info, get_v1_agents, get_v1_agent, post_v1_agent_install, @@ -360,6 +400,24 @@ pub async fn shutdown_servers(state: &Arc) { components( schemas( HealthResponse, + DesktopState, + DesktopResolution, + DesktopErrorInfo, + DesktopProcessInfo, + DesktopStatusResponse, + DesktopStartRequest, + DesktopScreenshotQuery, + DesktopRegionScreenshotQuery, + DesktopMousePositionResponse, + DesktopMouseButton, + DesktopMouseMoveRequest, + DesktopMouseClickRequest, + DesktopMouseDragRequest, + DesktopMouseScrollRequest, + DesktopKeyboardTypeRequest, + DesktopKeyboardPressRequest, + DesktopActionResponse, + DesktopDisplayInfoResponse, ServerStatus, ServerStatusInfo, AgentCapabilities, @@ -438,6 +496,12 @@ impl From for ApiError { } } +impl From for ApiError { + fn from(value: DesktopProblem) -> Self { + Self::Problem(value.to_problem_details()) + } +} + impl IntoResponse for ApiError { fn into_response(self) -> Response { let problem = match &self { @@ -476,6 +540,305 @@ async fn get_v1_health() -> Json { }) } +/// Get desktop runtime status. +/// +/// Returns the current desktop runtime state, dependency status, active +/// display metadata, and supervised process information. +#[utoipa::path( + get, + path = "/v1/desktop/status", + tag = "v1", + responses( + (status = 200, description = "Desktop runtime status", body = DesktopStatusResponse), + (status = 401, description = "Authentication required", body = ProblemDetails) + ) +)] +async fn get_v1_desktop_status( + State(state): State>, +) -> Result, ApiError> { + Ok(Json(state.desktop_runtime().status().await)) +} + +/// Start the private desktop runtime. +/// +/// Lazily launches the managed Xvfb/openbox stack, validates display health, +/// and returns the resulting desktop status snapshot. +#[utoipa::path( + post, + path = "/v1/desktop/start", + tag = "v1", + request_body = DesktopStartRequest, + responses( + (status = 200, description = "Desktop runtime status after start", body = DesktopStatusResponse), + (status = 400, description = "Invalid desktop start request", body = ProblemDetails), + (status = 409, description = "Desktop runtime is already transitioning", body = ProblemDetails), + (status = 501, description = "Desktop API unsupported on this platform", body = ProblemDetails), + (status = 503, description = "Desktop runtime could not be started", body = ProblemDetails) + ) +)] +async fn post_v1_desktop_start( + State(state): State>, + Json(body): Json, +) -> Result, ApiError> { + let status = state.desktop_runtime().start(body).await?; + Ok(Json(status)) +} + +/// Stop the private desktop runtime. +/// +/// Terminates the managed openbox/Xvfb/dbus processes owned by the desktop +/// runtime and returns the resulting status snapshot. +#[utoipa::path( + post, + path = "/v1/desktop/stop", + tag = "v1", + responses( + (status = 200, description = "Desktop runtime status after stop", body = DesktopStatusResponse), + (status = 409, description = "Desktop runtime is already transitioning", body = ProblemDetails) + ) +)] +async fn post_v1_desktop_stop( + State(state): State>, +) -> Result, ApiError> { + let status = state.desktop_runtime().stop().await?; + Ok(Json(status)) +} + +/// Capture a full desktop screenshot. +/// +/// Performs a health-gated full-frame screenshot of the managed desktop and +/// returns PNG bytes. +#[utoipa::path( + get, + path = "/v1/desktop/screenshot", + tag = "v1", + responses( + (status = 200, description = "Desktop screenshot as PNG bytes"), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 503, description = "Desktop runtime health or screenshot capture failed", body = ProblemDetails) + ) +)] +async fn get_v1_desktop_screenshot( + State(state): State>, +) -> Result { + let bytes = state.desktop_runtime().screenshot().await?; + Ok(([(header::CONTENT_TYPE, "image/png")], Bytes::from(bytes)).into_response()) +} + +/// Capture a desktop screenshot region. +/// +/// Performs a health-gated screenshot crop against the managed desktop and +/// returns the requested PNG region bytes. +#[utoipa::path( + get, + path = "/v1/desktop/screenshot/region", + tag = "v1", + params( + ("x" = i32, Query, description = "Region x coordinate"), + ("y" = i32, Query, description = "Region y coordinate"), + ("width" = u32, Query, description = "Region width"), + ("height" = u32, Query, description = "Region height") + ), + responses( + (status = 200, description = "Desktop screenshot region as PNG bytes"), + (status = 400, description = "Invalid screenshot region", body = ProblemDetails), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 503, description = "Desktop runtime health or screenshot capture failed", body = ProblemDetails) + ) +)] +async fn get_v1_desktop_screenshot_region( + State(state): State>, + Query(query): Query, +) -> Result { + let bytes = state.desktop_runtime().screenshot_region(query).await?; + Ok(([(header::CONTENT_TYPE, "image/png")], Bytes::from(bytes)).into_response()) +} + +/// Get the current desktop mouse position. +/// +/// Performs a health-gated mouse position query against the managed desktop. +#[utoipa::path( + get, + path = "/v1/desktop/mouse/position", + tag = "v1", + responses( + (status = 200, description = "Desktop mouse position", body = DesktopMousePositionResponse), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 503, description = "Desktop runtime health or input check failed", body = ProblemDetails) + ) +)] +async fn get_v1_desktop_mouse_position( + State(state): State>, +) -> Result, ApiError> { + let position = state.desktop_runtime().mouse_position().await?; + Ok(Json(position)) +} + +/// Move the desktop mouse. +/// +/// Performs a health-gated absolute pointer move on the managed desktop and +/// returns the resulting mouse position. +#[utoipa::path( + post, + path = "/v1/desktop/mouse/move", + tag = "v1", + request_body = DesktopMouseMoveRequest, + responses( + (status = 200, description = "Desktop mouse position after move", body = DesktopMousePositionResponse), + (status = 400, description = "Invalid mouse move request", body = ProblemDetails), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 503, description = "Desktop runtime health or input failed", body = ProblemDetails) + ) +)] +async fn post_v1_desktop_mouse_move( + State(state): State>, + Json(body): Json, +) -> Result, ApiError> { + let position = state.desktop_runtime().move_mouse(body).await?; + Ok(Json(position)) +} + +/// Click on the desktop. +/// +/// Performs a health-gated pointer move and click against the managed desktop +/// and returns the resulting mouse position. +#[utoipa::path( + post, + path = "/v1/desktop/mouse/click", + tag = "v1", + request_body = DesktopMouseClickRequest, + responses( + (status = 200, description = "Desktop mouse position after click", body = DesktopMousePositionResponse), + (status = 400, description = "Invalid mouse click request", body = ProblemDetails), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 503, description = "Desktop runtime health or input failed", body = ProblemDetails) + ) +)] +async fn post_v1_desktop_mouse_click( + State(state): State>, + Json(body): Json, +) -> Result, ApiError> { + let position = state.desktop_runtime().click_mouse(body).await?; + Ok(Json(position)) +} + +/// Drag the desktop mouse. +/// +/// Performs a health-gated drag gesture against the managed desktop and +/// returns the resulting mouse position. +#[utoipa::path( + post, + path = "/v1/desktop/mouse/drag", + tag = "v1", + request_body = DesktopMouseDragRequest, + responses( + (status = 200, description = "Desktop mouse position after drag", body = DesktopMousePositionResponse), + (status = 400, description = "Invalid mouse drag request", body = ProblemDetails), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 503, description = "Desktop runtime health or input failed", body = ProblemDetails) + ) +)] +async fn post_v1_desktop_mouse_drag( + State(state): State>, + Json(body): Json, +) -> Result, ApiError> { + let position = state.desktop_runtime().drag_mouse(body).await?; + Ok(Json(position)) +} + +/// Scroll the desktop mouse wheel. +/// +/// Performs a health-gated scroll gesture at the requested coordinates and +/// returns the resulting mouse position. +#[utoipa::path( + post, + path = "/v1/desktop/mouse/scroll", + tag = "v1", + request_body = DesktopMouseScrollRequest, + responses( + (status = 200, description = "Desktop mouse position after scroll", body = DesktopMousePositionResponse), + (status = 400, description = "Invalid mouse scroll request", body = ProblemDetails), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 503, description = "Desktop runtime health or input failed", body = ProblemDetails) + ) +)] +async fn post_v1_desktop_mouse_scroll( + State(state): State>, + Json(body): Json, +) -> Result, ApiError> { + let position = state.desktop_runtime().scroll_mouse(body).await?; + Ok(Json(position)) +} + +/// Type desktop keyboard text. +/// +/// Performs a health-gated `xdotool type` operation against the managed +/// desktop. +#[utoipa::path( + post, + path = "/v1/desktop/keyboard/type", + tag = "v1", + request_body = DesktopKeyboardTypeRequest, + responses( + (status = 200, description = "Desktop keyboard action result", body = DesktopActionResponse), + (status = 400, description = "Invalid keyboard type request", body = ProblemDetails), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 503, description = "Desktop runtime health or input failed", body = ProblemDetails) + ) +)] +async fn post_v1_desktop_keyboard_type( + State(state): State>, + Json(body): Json, +) -> Result, ApiError> { + let response = state.desktop_runtime().type_text(body).await?; + Ok(Json(response)) +} + +/// Press a desktop keyboard shortcut. +/// +/// Performs a health-gated `xdotool key` operation against the managed +/// desktop. +#[utoipa::path( + post, + path = "/v1/desktop/keyboard/press", + tag = "v1", + request_body = DesktopKeyboardPressRequest, + responses( + (status = 200, description = "Desktop keyboard action result", body = DesktopActionResponse), + (status = 400, description = "Invalid keyboard press request", body = ProblemDetails), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 503, description = "Desktop runtime health or input failed", body = ProblemDetails) + ) +)] +async fn post_v1_desktop_keyboard_press( + State(state): State>, + Json(body): Json, +) -> Result, ApiError> { + let response = state.desktop_runtime().press_key(body).await?; + Ok(Json(response)) +} + +/// Get desktop display information. +/// +/// Performs a health-gated display query against the managed desktop and +/// returns the current display identifier and resolution. +#[utoipa::path( + get, + path = "/v1/desktop/display/info", + tag = "v1", + responses( + (status = 200, description = "Desktop display information", body = DesktopDisplayInfoResponse), + (status = 409, description = "Desktop runtime is not ready", body = ProblemDetails), + (status = 503, description = "Desktop runtime health or display query failed", body = ProblemDetails) + ) +)] +async fn get_v1_desktop_display_info( + State(state): State>, +) -> Result, ApiError> { + let info = state.desktop_runtime().display_info().await?; + Ok(Json(info)) +} + #[utoipa::path( get, path = "/v1/agents", diff --git a/server/packages/sandbox-agent/tests/v1_api.rs b/server/packages/sandbox-agent/tests/v1_api.rs index fa572e6..70b28f9 100644 --- a/server/packages/sandbox-agent/tests/v1_api.rs +++ b/server/packages/sandbox-agent/tests/v1_api.rs @@ -50,6 +50,15 @@ struct EnvVarGuard { previous: Option, } +struct FakeDesktopEnv { + _temp: TempDir, + _path: EnvVarGuard, + _xdg_state_home: EnvVarGuard, + _assume_linux: EnvVarGuard, + _display_num: EnvVarGuard, + _fake_state_dir: EnvVarGuard, +} + struct LiveServer { address: SocketAddr, shutdown_tx: Option>, @@ -167,6 +176,153 @@ exit 0 ); } +fn setup_fake_desktop_env() -> FakeDesktopEnv { + let temp = tempfile::tempdir().expect("create fake desktop tempdir"); + let bin_dir = temp.path().join("bin"); + let xdg_state_home = temp.path().join("state"); + let fake_state_dir = temp.path().join("desktop-state"); + fs::create_dir_all(&bin_dir).expect("create fake desktop bin dir"); + fs::create_dir_all(&xdg_state_home).expect("create xdg state home"); + fs::create_dir_all(&fake_state_dir).expect("create fake state dir"); + + write_executable( + &bin_dir.join("Xvfb"), + r#"#!/usr/bin/env sh +set -eu +display="${1:-:99}" +number="${display#:}" +socket="/tmp/.X11-unix/X${number}" +mkdir -p /tmp/.X11-unix +touch "$socket" +cleanup() { + rm -f "$socket" + exit 0 +} +trap cleanup INT TERM EXIT +while :; do + sleep 1 +done +"#, + ); + + write_executable( + &bin_dir.join("openbox"), + r#"#!/usr/bin/env sh +set -eu +trap 'exit 0' INT TERM +while :; do + sleep 1 +done +"#, + ); + + write_executable( + &bin_dir.join("dbus-launch"), + r#"#!/usr/bin/env sh +set -eu +echo "DBUS_SESSION_BUS_ADDRESS=unix:path=/tmp/sandbox-agent-test-bus" +echo "DBUS_SESSION_BUS_PID=$$" +"#, + ); + + write_executable( + &bin_dir.join("xrandr"), + r#"#!/usr/bin/env sh +set -eu +cat <<'EOF' +Screen 0: minimum 1 x 1, current 1440 x 900, maximum 32767 x 32767 +EOF +"#, + ); + + write_executable( + &bin_dir.join("import"), + r#"#!/usr/bin/env sh +set -eu +printf '\211PNG\r\n\032\n\000\000\000\rIHDR\000\000\000\001\000\000\000\001\010\006\000\000\000\037\025\304\211\000\000\000\013IDATx\234c\000\001\000\000\005\000\001\r\n-\264\000\000\000\000IEND\256B`\202' +"#, + ); + + write_executable( + &bin_dir.join("xdotool"), + r#"#!/usr/bin/env sh +set -eu +state_dir="${SANDBOX_AGENT_DESKTOP_FAKE_STATE_DIR:?missing fake state dir}" +state_file="${state_dir}/mouse" +mkdir -p "$state_dir" +if [ ! -f "$state_file" ]; then + printf '0 0\n' > "$state_file" +fi + +read_state() { + read -r x y < "$state_file" +} + +write_state() { + printf '%s %s\n' "$1" "$2" > "$state_file" +} + +command="${1:-}" +case "$command" in + getmouselocation) + read_state + printf 'X=%s\nY=%s\nSCREEN=0\nWINDOW=0\n' "$x" "$y" + ;; + mousemove) + shift + x="${1:-0}" + y="${2:-0}" + shift 2 || true + while [ "$#" -gt 0 ]; do + token="$1" + shift + case "$token" in + mousemove) + x="${1:-0}" + y="${2:-0}" + shift 2 || true + ;; + mousedown|mouseup) + shift 1 || true + ;; + click) + if [ "${1:-}" = "--repeat" ]; then + shift 2 || true + fi + shift 1 || true + ;; + esac + done + write_state "$x" "$y" + ;; + type|key) + exit 0 + ;; + *) + exit 0 + ;; +esac +"#, + ); + + let original_path = std::env::var_os("PATH").unwrap_or_default(); + let mut paths = vec![bin_dir]; + paths.extend(std::env::split_paths(&original_path)); + let merged_path = std::env::join_paths(paths).expect("join PATH"); + + FakeDesktopEnv { + _temp: temp, + _path: EnvVarGuard::set_os("PATH", merged_path.as_os_str()), + _xdg_state_home: EnvVarGuard::set_os("XDG_STATE_HOME", xdg_state_home.as_os_str()), + _assume_linux: EnvVarGuard::set("SANDBOX_AGENT_DESKTOP_TEST_ASSUME_LINUX", "1"), + _display_num: EnvVarGuard::set("SANDBOX_AGENT_DESKTOP_DISPLAY_NUM", "190"), + _fake_state_dir: EnvVarGuard::set_os( + "SANDBOX_AGENT_DESKTOP_FAKE_STATE_DIR", + fake_state_dir.as_os_str(), + ), + } +} + fn serve_registry_once(document: Value) -> String { let listener = TcpListener::bind("127.0.0.1:0").expect("bind registry server"); let address = listener.local_addr().expect("registry address"); @@ -375,5 +531,7 @@ mod acp_transport; mod config_endpoints; #[path = "v1_api/control_plane.rs"] mod control_plane; +#[path = "v1_api/desktop.rs"] +mod desktop; #[path = "v1_api/processes.rs"] mod processes; diff --git a/server/packages/sandbox-agent/tests/v1_api/desktop.rs b/server/packages/sandbox-agent/tests/v1_api/desktop.rs new file mode 100644 index 0000000..c56e7e1 --- /dev/null +++ b/server/packages/sandbox-agent/tests/v1_api/desktop.rs @@ -0,0 +1,222 @@ +use super::*; +use serial_test::serial; + +#[tokio::test] +#[serial] +async fn v1_desktop_status_reports_install_required_when_dependencies_are_missing() { + let temp = tempfile::tempdir().expect("create empty path tempdir"); + let _path = EnvVarGuard::set_os("PATH", temp.path().as_os_str()); + let _assume_linux = EnvVarGuard::set("SANDBOX_AGENT_DESKTOP_TEST_ASSUME_LINUX", "1"); + + let test_app = TestApp::new(AuthConfig::disabled()); + + let (status, _, body) = send_request( + &test_app.app, + Method::GET, + "/v1/desktop/status", + None, + &[], + ) + .await; + + assert_eq!(status, StatusCode::OK); + let parsed = parse_json(&body); + assert_eq!(parsed["state"], "install_required"); + assert!(parsed["missingDependencies"] + .as_array() + .expect("missingDependencies array") + .iter() + .any(|value| value == "Xvfb")); + assert_eq!( + parsed["installCommand"], + "sandbox-agent install desktop --yes" + ); +} + +#[tokio::test] +#[serial] +async fn v1_desktop_lifecycle_and_actions_work_with_fake_runtime() { + let _fake = setup_fake_desktop_env(); + let test_app = TestApp::new(AuthConfig::disabled()); + + let (status, _, body) = send_request( + &test_app.app, + Method::POST, + "/v1/desktop/start", + Some(json!({ + "width": 1440, + "height": 900, + "dpi": 96 + })), + &[], + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "unexpected start response: {}", + String::from_utf8_lossy(&body) + ); + let parsed = parse_json(&body); + assert_eq!(parsed["state"], "active"); + let display = parsed["display"].as_str().expect("desktop display").to_string(); + assert!(display.starts_with(':')); + assert_eq!(parsed["resolution"]["width"], 1440); + assert_eq!(parsed["resolution"]["height"], 900); + + let (status, headers, body) = send_request_raw( + &test_app.app, + Method::GET, + "/v1/desktop/screenshot", + None, + &[], + None, + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()), + Some("image/png") + ); + assert!(body.starts_with(b"\x89PNG\r\n\x1a\n")); + + let (status, _, body) = send_request_raw( + &test_app.app, + Method::GET, + "/v1/desktop/screenshot/region?x=10&y=20&width=30&height=40", + None, + &[], + None, + ) + .await; + assert_eq!(status, StatusCode::OK); + assert!(body.starts_with(b"\x89PNG\r\n\x1a\n")); + + let (status, _, body) = send_request( + &test_app.app, + Method::GET, + "/v1/desktop/display/info", + None, + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + let display_info = parse_json(&body); + assert_eq!(display_info["display"], display); + assert_eq!(display_info["resolution"]["width"], 1440); + + let (status, _, body) = send_request( + &test_app.app, + Method::POST, + "/v1/desktop/mouse/move", + Some(json!({ "x": 400, "y": 300 })), + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + let mouse = parse_json(&body); + assert_eq!(mouse["x"], 400); + assert_eq!(mouse["y"], 300); + + let (status, _, body) = send_request( + &test_app.app, + Method::POST, + "/v1/desktop/mouse/drag", + Some(json!({ + "startX": 100, + "startY": 110, + "endX": 220, + "endY": 230, + "button": "left" + })), + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + let dragged = parse_json(&body); + assert_eq!(dragged["x"], 220); + assert_eq!(dragged["y"], 230); + + let (status, _, body) = send_request( + &test_app.app, + Method::POST, + "/v1/desktop/mouse/click", + Some(json!({ + "x": 220, + "y": 230, + "button": "left", + "clickCount": 1 + })), + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + let clicked = parse_json(&body); + assert_eq!(clicked["x"], 220); + assert_eq!(clicked["y"], 230); + + let (status, _, body) = send_request( + &test_app.app, + Method::POST, + "/v1/desktop/mouse/scroll", + Some(json!({ + "x": 220, + "y": 230, + "deltaY": -3 + })), + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + let scrolled = parse_json(&body); + assert_eq!(scrolled["x"], 220); + assert_eq!(scrolled["y"], 230); + + let (status, _, body) = send_request( + &test_app.app, + Method::GET, + "/v1/desktop/mouse/position", + None, + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + let position = parse_json(&body); + assert_eq!(position["x"], 220); + assert_eq!(position["y"], 230); + + let (status, _, body) = send_request( + &test_app.app, + Method::POST, + "/v1/desktop/keyboard/type", + Some(json!({ "text": "hello world", "delayMs": 5 })), + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(parse_json(&body)["ok"], true); + + let (status, _, body) = send_request( + &test_app.app, + Method::POST, + "/v1/desktop/keyboard/press", + Some(json!({ "key": "ctrl+l" })), + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(parse_json(&body)["ok"], true); + + let (status, _, body) = send_request( + &test_app.app, + Method::POST, + "/v1/desktop/stop", + None, + &[], + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(parse_json(&body)["state"], "inactive"); +}