feat: desktop computer-use APIs with windows, launch/open, and neko streaming

Adds desktop computer-use endpoints (windows, screenshots, mouse/keyboard,
launch/open), enhances neko-based streaming integration, updates inspector
UI with desktop debug tab, and adds common software test infrastructure.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Nathan Flurry 2026-03-17 02:35:52 -07:00
parent 2d8508d6e2
commit dff7614b11
17 changed files with 4045 additions and 136 deletions

View file

@ -628,6 +628,105 @@
}
}
},
"/v1/desktop/clipboard": {
"get": {
"tags": ["v1"],
"summary": "Read the desktop clipboard.",
"description": "Returns the current text content of the X11 clipboard.",
"operationId": "get_v1_desktop_clipboard",
"parameters": [
{
"name": "selection",
"in": "query",
"required": false,
"schema": {
"type": "string",
"nullable": true
}
}
],
"responses": {
"200": {
"description": "Clipboard contents",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DesktopClipboardResponse"
}
}
}
},
"409": {
"description": "Desktop runtime is not ready",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
}
},
"500": {
"description": "Clipboard read failed",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
}
}
}
},
"post": {
"tags": ["v1"],
"summary": "Write to the desktop clipboard.",
"description": "Sets the text content of the X11 clipboard.",
"operationId": "post_v1_desktop_clipboard",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DesktopClipboardWriteRequest"
}
}
},
"required": true
},
"responses": {
"200": {
"description": "Clipboard updated",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DesktopActionResponse"
}
}
}
},
"409": {
"description": "Desktop runtime is not ready",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
}
},
"500": {
"description": "Clipboard write failed",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
}
}
}
}
},
"/v1/desktop/display/info": {
"get": {
"tags": ["v1"],
@ -908,6 +1007,56 @@
}
}
},
"/v1/desktop/launch": {
"post": {
"tags": ["v1"],
"summary": "Launch a desktop application.",
"description": "Launches an application by name on the managed desktop, optionally waiting\nfor its window to appear.",
"operationId": "post_v1_desktop_launch",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DesktopLaunchRequest"
}
}
},
"required": true
},
"responses": {
"200": {
"description": "Application launched",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DesktopLaunchResponse"
}
}
}
},
"404": {
"description": "Application not found",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
}
},
"409": {
"description": "Desktop runtime is not ready",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
}
}
}
}
},
"/v1/desktop/mouse/click": {
"post": {
"tags": ["v1"],
@ -1308,6 +1457,46 @@
}
}
},
"/v1/desktop/open": {
"post": {
"tags": ["v1"],
"summary": "Open a file or URL with the default handler.",
"description": "Opens a file path or URL using xdg-open on the managed desktop.",
"operationId": "post_v1_desktop_open",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DesktopOpenRequest"
}
}
},
"required": true
},
"responses": {
"200": {
"description": "Target opened",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DesktopOpenResponse"
}
}
}
},
"409": {
"description": "Desktop runtime is not ready",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
}
}
}
}
},
"/v1/desktop/recording/start": {
"post": {
"tags": ["v1"],
@ -1585,6 +1774,15 @@
"format": "float",
"nullable": true
}
},
{
"name": "showCursor",
"in": "query",
"required": false,
"schema": {
"type": "boolean",
"nullable": true
}
}
],
"responses": {
@ -1702,6 +1900,15 @@
"format": "float",
"nullable": true
}
},
{
"name": "showCursor",
"in": "query",
"required": false,
"schema": {
"type": "boolean",
"nullable": true
}
}
],
"responses": {
@ -1871,51 +2078,11 @@
}
}
},
"/v1/desktop/stream/start": {
"post": {
"tags": ["v1"],
"summary": "Start desktop streaming.",
"description": "Enables desktop websocket streaming for the managed desktop.",
"operationId": "post_v1_desktop_stream_start",
"responses": {
"200": {
"description": "Desktop streaming started",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DesktopStreamStatusResponse"
}
}
}
}
}
}
},
"/v1/desktop/stream/stop": {
"post": {
"tags": ["v1"],
"summary": "Stop desktop streaming.",
"description": "Disables desktop websocket streaming for the managed desktop.",
"operationId": "post_v1_desktop_stream_stop",
"responses": {
"200": {
"description": "Desktop streaming stopped",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DesktopStreamStatusResponse"
}
}
}
}
}
}
},
"/v1/desktop/stream/ws": {
"/v1/desktop/stream/signaling": {
"get": {
"tags": ["v1"],
"summary": "Open a desktop websocket streaming session.",
"description": "Upgrades the connection to a websocket that streams JPEG desktop frames and\naccepts mouse and keyboard control frames.",
"summary": "Open a desktop WebRTC signaling session.",
"description": "Upgrades the connection to a WebSocket used for WebRTC signaling between\nthe browser client and the desktop streaming process. Also accepts mouse\nand keyboard input frames as a fallback transport.",
"operationId": "get_v1_desktop_stream_ws",
"parameters": [
{
@ -1956,6 +2123,66 @@
}
}
},
"/v1/desktop/stream/start": {
"post": {
"tags": ["v1"],
"summary": "Start desktop streaming.",
"description": "Enables desktop websocket streaming for the managed desktop.",
"operationId": "post_v1_desktop_stream_start",
"responses": {
"200": {
"description": "Desktop streaming started",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DesktopStreamStatusResponse"
}
}
}
}
}
}
},
"/v1/desktop/stream/status": {
"get": {
"tags": ["v1"],
"summary": "Get desktop stream status.",
"description": "Returns the current state of the desktop WebRTC streaming session.",
"operationId": "get_v1_desktop_stream_status",
"responses": {
"200": {
"description": "Desktop stream status",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DesktopStreamStatusResponse"
}
}
}
}
}
}
},
"/v1/desktop/stream/stop": {
"post": {
"tags": ["v1"],
"summary": "Stop desktop streaming.",
"description": "Disables desktop websocket streaming for the managed desktop.",
"operationId": "post_v1_desktop_stream_stop",
"responses": {
"200": {
"description": "Desktop streaming stopped",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DesktopStreamStatusResponse"
}
}
}
}
}
}
},
"/v1/desktop/windows": {
"get": {
"tags": ["v1"],
@ -1996,6 +2223,219 @@
}
}
},
"/v1/desktop/windows/focused": {
"get": {
"tags": ["v1"],
"summary": "Get the currently focused desktop window.",
"description": "Returns information about the window that currently has input focus.",
"operationId": "get_v1_desktop_windows_focused",
"responses": {
"200": {
"description": "Focused window info",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DesktopWindowInfo"
}
}
}
},
"404": {
"description": "No window is focused",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
}
},
"409": {
"description": "Desktop runtime is not ready",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
}
}
}
}
},
"/v1/desktop/windows/{id}/focus": {
"post": {
"tags": ["v1"],
"summary": "Focus a desktop window.",
"description": "Brings the specified window to the foreground and gives it input focus.",
"operationId": "post_v1_desktop_window_focus",
"parameters": [
{
"name": "id",
"in": "path",
"description": "X11 window ID",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Window info after focus",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DesktopWindowInfo"
}
}
}
},
"404": {
"description": "Window not found",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
}
},
"409": {
"description": "Desktop runtime is not ready",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
}
}
}
}
},
"/v1/desktop/windows/{id}/move": {
"post": {
"tags": ["v1"],
"summary": "Move a desktop window.",
"description": "Moves the specified window to the given position.",
"operationId": "post_v1_desktop_window_move",
"parameters": [
{
"name": "id",
"in": "path",
"description": "X11 window ID",
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DesktopWindowMoveRequest"
}
}
},
"required": true
},
"responses": {
"200": {
"description": "Window info after move",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DesktopWindowInfo"
}
}
}
},
"404": {
"description": "Window not found",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
}
},
"409": {
"description": "Desktop runtime is not ready",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
}
}
}
}
},
"/v1/desktop/windows/{id}/resize": {
"post": {
"tags": ["v1"],
"summary": "Resize a desktop window.",
"description": "Resizes the specified window to the given dimensions.",
"operationId": "post_v1_desktop_window_resize",
"parameters": [
{
"name": "id",
"in": "path",
"description": "X11 window ID",
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DesktopWindowResizeRequest"
}
}
},
"required": true
},
"responses": {
"200": {
"description": "Window info after resize",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DesktopWindowInfo"
}
}
}
},
"404": {
"description": "Window not found",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
}
},
"409": {
"description": "Desktop runtime is not ready",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ProblemDetails"
}
}
}
}
}
}
},
"/v1/fs/entries": {
"get": {
"tags": ["v1"],
@ -3326,6 +3766,40 @@
}
}
},
"DesktopClipboardQuery": {
"type": "object",
"properties": {
"selection": {
"type": "string",
"nullable": true
}
}
},
"DesktopClipboardResponse": {
"type": "object",
"required": ["text", "selection"],
"properties": {
"selection": {
"type": "string"
},
"text": {
"type": "string"
}
}
},
"DesktopClipboardWriteRequest": {
"type": "object",
"required": ["text"],
"properties": {
"selection": {
"type": "string",
"nullable": true
},
"text": {
"type": "string"
}
}
},
"DesktopDisplayInfoResponse": {
"type": "object",
"required": ["display", "resolution"],
@ -3421,6 +3895,45 @@
}
}
},
"DesktopLaunchRequest": {
"type": "object",
"required": ["app"],
"properties": {
"app": {
"type": "string"
},
"args": {
"type": "array",
"items": {
"type": "string"
},
"nullable": true
},
"wait": {
"type": "boolean",
"nullable": true
}
}
},
"DesktopLaunchResponse": {
"type": "object",
"required": ["processId"],
"properties": {
"pid": {
"type": "integer",
"format": "int32",
"nullable": true,
"minimum": 0
},
"processId": {
"type": "string"
},
"windowId": {
"type": "string",
"nullable": true
}
}
},
"DesktopMouseButton": {
"type": "string",
"enum": ["left", "middle", "right"]
@ -3590,6 +4103,30 @@
}
}
},
"DesktopOpenRequest": {
"type": "object",
"required": ["target"],
"properties": {
"target": {
"type": "string"
}
}
},
"DesktopOpenResponse": {
"type": "object",
"required": ["processId"],
"properties": {
"pid": {
"type": "integer",
"format": "int32",
"nullable": true,
"minimum": 0
},
"processId": {
"type": "string"
}
}
},
"DesktopProcessInfo": {
"type": "object",
"required": ["name", "running"],
@ -3698,6 +4235,10 @@
"format": "float",
"nullable": true
},
"showCursor": {
"type": "boolean",
"nullable": true
},
"width": {
"type": "integer",
"format": "int32",
@ -3760,12 +4301,21 @@
"type": "number",
"format": "float",
"nullable": true
},
"showCursor": {
"type": "boolean",
"nullable": true
}
}
},
"DesktopStartRequest": {
"type": "object",
"properties": {
"displayNum": {
"type": "integer",
"format": "int32",
"nullable": true
},
"dpi": {
"type": "integer",
"format": "int32",
@ -3778,6 +4328,34 @@
"nullable": true,
"minimum": 0
},
"recordingFps": {
"type": "integer",
"format": "int32",
"nullable": true,
"minimum": 0
},
"stateDir": {
"type": "string",
"nullable": true
},
"streamAudioCodec": {
"type": "string",
"nullable": true
},
"streamFrameRate": {
"type": "integer",
"format": "int32",
"nullable": true,
"minimum": 0
},
"streamVideoCodec": {
"type": "string",
"nullable": true
},
"webrtcPortRange": {
"type": "string",
"nullable": true
},
"width": {
"type": "integer",
"format": "int32",
@ -3840,6 +4418,13 @@
},
"state": {
"$ref": "#/components/schemas/DesktopState"
},
"windows": {
"type": "array",
"items": {
"$ref": "#/components/schemas/DesktopWindowInfo"
},
"description": "Current visible windows (included when the desktop is active)."
}
}
},
@ -3849,6 +4434,14 @@
"properties": {
"active": {
"type": "boolean"
},
"processId": {
"type": "string",
"nullable": true
},
"windowId": {
"type": "string",
"nullable": true
}
}
},
@ -3897,6 +4490,36 @@
}
}
},
"DesktopWindowMoveRequest": {
"type": "object",
"required": ["x", "y"],
"properties": {
"x": {
"type": "integer",
"format": "int32"
},
"y": {
"type": "integer",
"format": "int32"
}
}
},
"DesktopWindowResizeRequest": {
"type": "object",
"required": ["width", "height"],
"properties": {
"height": {
"type": "integer",
"format": "int32",
"minimum": 0
},
"width": {
"type": "integer",
"format": "int32",
"minimum": 0
}
}
},
"ErrorType": {
"type": "string",
"enum": [