diff --git a/docker/inspector-dev/Dockerfile b/docker/inspector-dev/Dockerfile new file mode 100644 index 0000000..b55923f --- /dev/null +++ b/docker/inspector-dev/Dockerfile @@ -0,0 +1,7 @@ +FROM node:22-bookworm-slim + +RUN npm install -g pnpm@10.28.2 + +WORKDIR /app + +CMD ["bash", "-lc", "pnpm install --filter @sandbox-agent/inspector... && cd frontend/packages/inspector && exec pnpm vite --host 0.0.0.0 --port 5173"] diff --git a/docker/test-agent/Dockerfile b/docker/test-agent/Dockerfile index dab4391..67888b3 100644 --- a/docker/test-agent/Dockerfile +++ b/docker/test-agent/Dockerfile @@ -6,6 +6,7 @@ COPY server/ ./server/ COPY gigacode/ ./gigacode/ COPY resources/agent-schemas/artifacts/ ./resources/agent-schemas/artifacts/ COPY scripts/agent-configs/ ./scripts/agent-configs/ +COPY scripts/audit-acp-deps/ ./scripts/audit-acp-deps/ ENV SANDBOX_AGENT_SKIP_INSPECTOR=1 @@ -15,6 +16,12 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ cargo build -p sandbox-agent --release && \ cp target/release/sandbox-agent /sandbox-agent +# Extract neko binary from the official image for WebRTC desktop streaming. +# Using neko v3 base image from GHCR which provides multi-arch support (amd64, arm64). +# Pinned by digest to prevent breaking changes from upstream. +# Reference client: https://github.com/demodesk/neko-client/blob/37f93eae6bd55b333c94bd009d7f2b079075a026/src/component/internal/webrtc.ts +FROM ghcr.io/m1k1o/neko/base@sha256:0c384afa56268aaa2d5570211d284763d0840dcdd1a7d9a24be3081d94d3dfce AS neko-base + FROM node:22-bookworm-slim RUN apt-get update -qq && \ apt-get install -y -qq --no-install-recommends \ @@ -26,6 +33,15 @@ RUN apt-get update -qq && \ xdotool \ imagemagick \ ffmpeg \ + gstreamer1.0-tools \ + gstreamer1.0-plugins-base \ + gstreamer1.0-plugins-good \ + gstreamer1.0-plugins-bad \ + gstreamer1.0-plugins-ugly \ + gstreamer1.0-nice \ + gstreamer1.0-x \ + gstreamer1.0-pulseaudio \ + libxcvt0 \ x11-xserver-utils \ dbus-x11 \ xauth \ @@ -35,8 +51,11 @@ RUN apt-get update -qq && \ rm -rf /var/lib/apt/lists/* COPY --from=builder /sandbox-agent /usr/local/bin/sandbox-agent +COPY --from=neko-base /usr/bin/neko /usr/local/bin/neko EXPOSE 3000 +# Expose UDP port range for WebRTC media transport +EXPOSE 59050-59070/udp ENTRYPOINT ["/usr/local/bin/sandbox-agent"] CMD ["server", "--host", "0.0.0.0", "--port", "3000", "--no-token"] diff --git a/docs/common-software.mdx b/docs/common-software.mdx new file mode 100644 index 0000000..7997a92 --- /dev/null +++ b/docs/common-software.mdx @@ -0,0 +1,560 @@ +--- +title: "Common Software" +description: "Install browsers, languages, databases, and other tools inside the sandbox." +sidebarTitle: "Common Software" +icon: "box-open" +--- + +The sandbox runs a Debian/Ubuntu base image. You can install software with `apt-get` via the [Process API](/processes) or by customizing your Docker image. This page covers commonly needed packages and how to install them. + +## Browsers + +### Chromium + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "chromium", "chromium-sandbox"], +}); + +// Launch headless +await sdk.runProcess({ + command: "chromium", + args: ["--headless", "--no-sandbox", "--disable-gpu", "https://example.com"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","chromium","chromium-sandbox"]}' +``` + + + +Use `--no-sandbox` when running Chromium inside a container. The container itself provides isolation. + + +### Firefox + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "firefox-esr"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","firefox-esr"]}' +``` + + +### Playwright browsers + +Playwright bundles its own browser binaries. Install the Playwright CLI and let it download browsers for you. + + +```ts TypeScript +await sdk.runProcess({ + command: "npx", + args: ["playwright", "install", "--with-deps", "chromium"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"npx","args":["playwright","install","--with-deps","chromium"]}' +``` + + +--- + +## Languages and runtimes + +### Node.js + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "nodejs", "npm"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","nodejs","npm"]}' +``` + + +For a specific version, use [nvm](https://github.com/nvm-sh/nvm): + +```ts TypeScript +await sdk.runProcess({ + command: "bash", + args: ["-c", "curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash && . ~/.nvm/nvm.sh && nvm install 22"], +}); +``` + +### Python + +Python 3 is typically pre-installed. To add pip and common packages: + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "python3", "python3-pip", "python3-venv"], +}); + +await sdk.runProcess({ + command: "pip3", + args: ["install", "numpy", "pandas", "matplotlib"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","python3","python3-pip","python3-venv"]}' + +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"pip3","args":["install","numpy","pandas","matplotlib"]}' +``` + + +### Go + + +```ts TypeScript +await sdk.runProcess({ + command: "bash", + args: ["-c", "curl -fsSL https://go.dev/dl/go1.23.6.linux-amd64.tar.gz | tar -C /usr/local -xz"], +}); + +// Add to PATH for subsequent commands +await sdk.runProcess({ + command: "bash", + args: ["-c", "export PATH=$PATH:/usr/local/go/bin && go version"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"bash","args":["-c","curl -fsSL https://go.dev/dl/go1.23.6.linux-amd64.tar.gz | tar -C /usr/local -xz"]}' +``` + + +### Rust + + +```ts TypeScript +await sdk.runProcess({ + command: "bash", + args: ["-c", "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"bash","args":["-c","curl --proto =https --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y"]}' +``` + + +### Java (OpenJDK) + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "default-jdk"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","default-jdk"]}' +``` + + +### Ruby + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "ruby-full"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","ruby-full"]}' +``` + + +--- + +## Databases + +### PostgreSQL + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "postgresql", "postgresql-client"], +}); + +// Start the service +const proc = await sdk.createProcess({ + command: "bash", + args: ["-c", "su - postgres -c 'pg_ctlcluster 15 main start'"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","postgresql","postgresql-client"]}' +``` + + +### SQLite + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "sqlite3"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","sqlite3"]}' +``` + + +### Redis + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "redis-server"], +}); + +const proc = await sdk.createProcess({ + command: "redis-server", + args: ["--daemonize", "no"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","redis-server"]}' + +curl -X POST "http://127.0.0.1:2468/v1/processes" \ + -H "Content-Type: application/json" \ + -d '{"command":"redis-server","args":["--daemonize","no"]}' +``` + + +### MySQL / MariaDB + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "mariadb-server", "mariadb-client"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","mariadb-server","mariadb-client"]}' +``` + + +--- + +## Build tools + +### Essential build toolchain + +Most compiled software needs the standard build toolchain: + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "build-essential", "cmake", "pkg-config"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","build-essential","cmake","pkg-config"]}' +``` + + +This installs `gcc`, `g++`, `make`, `cmake`, and related tools. + +--- + +## Desktop applications + +These require the [Computer Use](/computer-use) desktop to be started first. + +### LibreOffice + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "libreoffice"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","libreoffice"]}' +``` + + +### GIMP + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "gimp"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","gimp"]}' +``` + + +### VLC + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "vlc"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","vlc"]}' +``` + + +### VS Code (code-server) + + +```ts TypeScript +await sdk.runProcess({ + command: "bash", + args: ["-c", "curl -fsSL https://code-server.dev/install.sh | sh"], +}); + +const proc = await sdk.createProcess({ + command: "code-server", + args: ["--bind-addr", "0.0.0.0:8080", "--auth", "none"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"bash","args":["-c","curl -fsSL https://code-server.dev/install.sh | sh"]}' + +curl -X POST "http://127.0.0.1:2468/v1/processes" \ + -H "Content-Type: application/json" \ + -d '{"command":"code-server","args":["--bind-addr","0.0.0.0:8080","--auth","none"]}' +``` + + +--- + +## CLI tools + +### Git + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "git"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","git"]}' +``` + + +### Docker + + +```ts TypeScript +await sdk.runProcess({ + command: "bash", + args: ["-c", "curl -fsSL https://get.docker.com | sh"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"bash","args":["-c","curl -fsSL https://get.docker.com | sh"]}' +``` + + +### jq + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "jq"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","jq"]}' +``` + + +### tmux + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "tmux"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","tmux"]}' +``` + + +--- + +## Media and graphics + +### FFmpeg + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "ffmpeg"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","ffmpeg"]}' +``` + + +### ImageMagick + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "imagemagick"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","imagemagick"]}' +``` + + +### Poppler (PDF utilities) + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "poppler-utils"], +}); + +// Convert PDF to images +await sdk.runProcess({ + command: "pdftoppm", + args: ["-png", "document.pdf", "output"], +}); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","poppler-utils"]}' +``` + + +--- + +## Pre-installing in a Docker image + +For production use, install software in your Dockerfile instead of at runtime. This avoids repeated downloads and makes startup faster. + +```dockerfile +FROM ubuntu:22.04 + +RUN apt-get update && apt-get install -y \ + chromium \ + firefox-esr \ + nodejs npm \ + python3 python3-pip \ + git curl wget \ + build-essential \ + sqlite3 \ + ffmpeg \ + imagemagick \ + jq \ + && rm -rf /var/lib/apt/lists/* + +RUN pip3 install numpy pandas matplotlib +``` + +See [Docker deployment](/deploy/docker) for how to use custom images with Sandbox Agent. diff --git a/docs/computer-use.mdx b/docs/computer-use.mdx new file mode 100644 index 0000000..63157cd --- /dev/null +++ b/docs/computer-use.mdx @@ -0,0 +1,553 @@ +--- +title: "Computer Use" +description: "Control a virtual desktop inside the sandbox with mouse, keyboard, screenshots, recordings, and live streaming." +sidebarTitle: "Computer Use" +icon: "desktop" +--- + +Sandbox Agent provides a managed virtual desktop (Xvfb + openbox) that you can control programmatically. This is useful for browser automation, GUI testing, and AI computer-use workflows. + +## Start and stop + + +```ts TypeScript +import { SandboxAgent } from "sandbox-agent"; + +const sdk = await SandboxAgent.connect({ + baseUrl: "http://127.0.0.1:2468", +}); + +const status = await sdk.startDesktop({ + width: 1920, + height: 1080, + dpi: 96, +}); + +console.log(status.state); // "active" +console.log(status.display); // ":99" + +// When done +await sdk.stopDesktop(); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/desktop/start" \ + -H "Content-Type: application/json" \ + -d '{"width":1920,"height":1080,"dpi":96}' + +curl -X POST "http://127.0.0.1:2468/v1/desktop/stop" +``` + + +All fields in the start request are optional. Defaults are 1440x900 at 96 DPI. + +## Status + + +```ts TypeScript +const status = await sdk.getDesktopStatus(); +console.log(status.state); // "inactive" | "active" | "failed" | ... +``` + +```bash cURL +curl "http://127.0.0.1:2468/v1/desktop/status" +``` + + +## Screenshots + +Capture the full desktop or a specific region. + + +```ts TypeScript +// Full screenshot (PNG by default) +const png = await sdk.takeDesktopScreenshot(); + +// JPEG at 70% quality, half scale +const jpeg = await sdk.takeDesktopScreenshot({ + format: "jpeg", + quality: 70, + scale: 0.5, +}); + +// Region screenshot +const region = await sdk.takeDesktopRegionScreenshot({ + x: 100, + y: 100, + width: 400, + height: 300, +}); +``` + +```bash cURL +curl "http://127.0.0.1:2468/v1/desktop/screenshot" --output screenshot.png + +curl "http://127.0.0.1:2468/v1/desktop/screenshot?format=jpeg&quality=70&scale=0.5" \ + --output screenshot.jpg + +curl "http://127.0.0.1:2468/v1/desktop/screenshot/region?x=100&y=100&width=400&height=300" \ + --output region.png +``` + + +## Mouse + + +```ts TypeScript +// Get current position +const pos = await sdk.getDesktopMousePosition(); +console.log(pos.x, pos.y); + +// Move +await sdk.moveDesktopMouse({ x: 500, y: 300 }); + +// Click (left by default) +await sdk.clickDesktop({ x: 500, y: 300 }); + +// Right click +await sdk.clickDesktop({ x: 500, y: 300, button: "right" }); + +// Double click +await sdk.clickDesktop({ x: 500, y: 300, clickCount: 2 }); + +// Drag +await sdk.dragDesktopMouse({ + startX: 100, startY: 100, + endX: 400, endY: 400, +}); + +// Scroll +await sdk.scrollDesktop({ x: 500, y: 300, deltaY: -3 }); +``` + +```bash cURL +curl "http://127.0.0.1:2468/v1/desktop/mouse/position" + +curl -X POST "http://127.0.0.1:2468/v1/desktop/mouse/click" \ + -H "Content-Type: application/json" \ + -d '{"x":500,"y":300}' + +curl -X POST "http://127.0.0.1:2468/v1/desktop/mouse/drag" \ + -H "Content-Type: application/json" \ + -d '{"startX":100,"startY":100,"endX":400,"endY":400}' + +curl -X POST "http://127.0.0.1:2468/v1/desktop/mouse/scroll" \ + -H "Content-Type: application/json" \ + -d '{"x":500,"y":300,"deltaY":-3}' +``` + + +## Keyboard + + +```ts TypeScript +// Type text +await sdk.typeDesktopText({ text: "Hello, world!" }); + +// Press a key with modifiers +await sdk.pressDesktopKey({ + key: "c", + modifiers: { ctrl: true }, +}); + +// Low-level key down/up +await sdk.keyDownDesktop({ key: "Shift_L" }); +await sdk.keyUpDesktop({ key: "Shift_L" }); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/desktop/keyboard/type" \ + -H "Content-Type: application/json" \ + -d '{"text":"Hello, world!"}' + +curl -X POST "http://127.0.0.1:2468/v1/desktop/keyboard/press" \ + -H "Content-Type: application/json" \ + -d '{"key":"c","modifiers":{"ctrl":true}}' +``` + + +## Display and windows + + +```ts TypeScript +const display = await sdk.getDesktopDisplayInfo(); +console.log(display.resolution); // { width: 1920, height: 1080, dpi: 96 } + +const { windows } = await sdk.listDesktopWindows(); +for (const win of windows) { + console.log(win.title, win.x, win.y, win.width, win.height); +} +``` + +```bash cURL +curl "http://127.0.0.1:2468/v1/desktop/display/info" + +curl "http://127.0.0.1:2468/v1/desktop/windows" +``` + + +## Recording + +Record the desktop to MP4. + + +```ts TypeScript +const recording = await sdk.startDesktopRecording({ fps: 30 }); +console.log(recording.id); + +// ... do things ... + +const stopped = await sdk.stopDesktopRecording(); + +// List all recordings +const { recordings } = await sdk.listDesktopRecordings(); + +// Download +const mp4 = await sdk.downloadDesktopRecording(recording.id); + +// Clean up +await sdk.deleteDesktopRecording(recording.id); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/desktop/recording/start" \ + -H "Content-Type: application/json" \ + -d '{"fps":30}' + +curl -X POST "http://127.0.0.1:2468/v1/desktop/recording/stop" + +curl "http://127.0.0.1:2468/v1/desktop/recordings" + +curl "http://127.0.0.1:2468/v1/desktop/recordings/rec_1/download" --output recording.mp4 + +curl -X DELETE "http://127.0.0.1:2468/v1/desktop/recordings/rec_1" +``` + + +## Desktop processes + +The desktop runtime manages several background processes (Xvfb, openbox, neko, ffmpeg). These are all registered with the general [Process API](/processes) under the `desktop` owner, so you can inspect logs, check status, and troubleshoot using the same tools you use for any other managed process. + + +```ts TypeScript +// List all processes, including desktop-owned ones +const { processes } = await sdk.listProcesses(); + +const desktopProcs = processes.filter((p) => p.owner === "desktop"); +for (const p of desktopProcs) { + console.log(p.id, p.command, p.status); +} + +// Read logs from a specific desktop process +const logs = await sdk.getProcessLogs(desktopProcs[0].id, { tail: 50 }); +for (const entry of logs.entries) { + console.log(entry.stream, atob(entry.data)); +} +``` + +```bash cURL +# List all processes (desktop processes have owner: "desktop") +curl "http://127.0.0.1:2468/v1/processes" + +# Get logs from a specific desktop process +curl "http://127.0.0.1:2468/v1/processes/proc_1/logs?tail=50" +``` + + +The desktop status endpoint also includes a summary of running processes: + + +```ts TypeScript +const status = await sdk.getDesktopStatus(); +for (const proc of status.processes) { + console.log(proc.name, proc.pid, proc.running); +} +``` + +```bash cURL +curl "http://127.0.0.1:2468/v1/desktop/status" +# Response includes: processes: [{ name: "Xvfb", pid: 123, running: true }, ...] +``` + + +| Process | Role | Restart policy | +|---------|------|---------------| +| Xvfb | Virtual X11 framebuffer | Auto-restart while desktop is active | +| openbox | Window manager | Auto-restart while desktop is active | +| neko | WebRTC streaming server (started by `startDesktopStream`) | No auto-restart | +| ffmpeg | Screen recorder (started by `startDesktopRecording`) | No auto-restart | + +## Live streaming + +Start a WebRTC stream for real-time desktop viewing in a browser. + + +```ts TypeScript +await sdk.startDesktopStream(); + +// Connect via the React DesktopViewer component or +// use the WebSocket signaling endpoint directly +// at ws://127.0.0.1:2468/v1/desktop/stream/signaling + +await sdk.stopDesktopStream(); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/desktop/stream/start" + +# Connect to ws://127.0.0.1:2468/v1/desktop/stream/signaling for WebRTC signaling + +curl -X POST "http://127.0.0.1:2468/v1/desktop/stream/stop" +``` + + +For a drop-in React component, see [React Components](/react-components). + +## Customizing the desktop environment + +The desktop runs inside the sandbox filesystem, so you can customize it using the [File System](/file-system) API before or after starting the desktop. The desktop HOME directory is located at `~/.local/state/sandbox-agent/desktop/home` (or `$XDG_STATE_HOME/sandbox-agent/desktop/home` if `XDG_STATE_HOME` is set). + +All configuration files below are written to paths relative to this HOME directory. + +### Window manager (openbox) + +The desktop uses [openbox](http://openbox.org/) as its window manager. You can customize its behavior, theme, and keyboard shortcuts by writing an `rc.xml` config file. + + +```ts TypeScript +const openboxConfig = ` + + + Clearlooks + NLIMC + DejaVu Sans10 + + 1 + + + + +`; + +await sdk.mkdirFs({ path: "~/.local/state/sandbox-agent/desktop/home/.config/openbox" }); +await sdk.writeFsFile( + { path: "~/.local/state/sandbox-agent/desktop/home/.config/openbox/rc.xml" }, + openboxConfig, +); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/fs/mkdir?path=~/.local/state/sandbox-agent/desktop/home/.config/openbox" + +curl -X PUT "http://127.0.0.1:2468/v1/fs/file?path=~/.local/state/sandbox-agent/desktop/home/.config/openbox/rc.xml" \ + -H "Content-Type: application/octet-stream" \ + --data-binary @rc.xml +``` + + +### Autostart programs + +Openbox runs scripts in `~/.config/openbox/autostart` on startup. Use this to launch applications, set the background, or configure the environment. + + +```ts TypeScript +const autostart = `#!/bin/sh +# Set a solid background color +xsetroot -solid "#1e1e2e" & + +# Launch a terminal +xterm -geometry 120x40+50+50 & + +# Launch a browser +firefox --no-remote & +`; + +await sdk.mkdirFs({ path: "~/.local/state/sandbox-agent/desktop/home/.config/openbox" }); +await sdk.writeFsFile( + { path: "~/.local/state/sandbox-agent/desktop/home/.config/openbox/autostart" }, + autostart, +); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/fs/mkdir?path=~/.local/state/sandbox-agent/desktop/home/.config/openbox" + +curl -X PUT "http://127.0.0.1:2468/v1/fs/file?path=~/.local/state/sandbox-agent/desktop/home/.config/openbox/autostart" \ + -H "Content-Type: application/octet-stream" \ + --data-binary @autostart.sh +``` + + + +The autostart script runs when openbox starts, which happens during `startDesktop()`. Write the autostart file before calling `startDesktop()` for it to take effect. + + +### Background + +There is no wallpaper set by default (the background is the X root window default). You can set it using `xsetroot` in the autostart script (as shown above), or use `feh` if you need an image: + + +```ts TypeScript +// Upload a wallpaper image +import fs from "node:fs"; + +const wallpaper = await fs.promises.readFile("./wallpaper.png"); +await sdk.writeFsFile( + { path: "~/.local/state/sandbox-agent/desktop/home/wallpaper.png" }, + wallpaper, +); + +// Set the autostart to apply it +const autostart = `#!/bin/sh +feh --bg-fill ~/wallpaper.png & +`; + +await sdk.mkdirFs({ path: "~/.local/state/sandbox-agent/desktop/home/.config/openbox" }); +await sdk.writeFsFile( + { path: "~/.local/state/sandbox-agent/desktop/home/.config/openbox/autostart" }, + autostart, +); +``` + +```bash cURL +curl -X PUT "http://127.0.0.1:2468/v1/fs/file?path=~/.local/state/sandbox-agent/desktop/home/wallpaper.png" \ + -H "Content-Type: application/octet-stream" \ + --data-binary @wallpaper.png + +curl -X PUT "http://127.0.0.1:2468/v1/fs/file?path=~/.local/state/sandbox-agent/desktop/home/.config/openbox/autostart" \ + -H "Content-Type: application/octet-stream" \ + --data-binary @autostart.sh +``` + + + +`feh` is not installed by default. Install it via the [Process API](/processes) before starting the desktop: `await sdk.runProcess({ command: "apt-get", args: ["install", "-y", "feh"] })`. + + +### Fonts + +Only `fonts-dejavu-core` is installed by default. To add more fonts, install them with your system package manager or copy font files into the sandbox: + + +```ts TypeScript +// Install a font package +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "fonts-noto", "fonts-liberation"], +}); + +// Or copy a custom font file +import fs from "node:fs"; + +const font = await fs.promises.readFile("./CustomFont.ttf"); +await sdk.mkdirFs({ path: "~/.local/state/sandbox-agent/desktop/home/.local/share/fonts" }); +await sdk.writeFsFile( + { path: "~/.local/state/sandbox-agent/desktop/home/.local/share/fonts/CustomFont.ttf" }, + font, +); + +// Rebuild the font cache +await sdk.runProcess({ command: "fc-cache", args: ["-fv"] }); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","fonts-noto","fonts-liberation"]}' + +curl -X POST "http://127.0.0.1:2468/v1/fs/mkdir?path=~/.local/state/sandbox-agent/desktop/home/.local/share/fonts" + +curl -X PUT "http://127.0.0.1:2468/v1/fs/file?path=~/.local/state/sandbox-agent/desktop/home/.local/share/fonts/CustomFont.ttf" \ + -H "Content-Type: application/octet-stream" \ + --data-binary @CustomFont.ttf + +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"fc-cache","args":["-fv"]}' +``` + + +### Cursor theme + + +```ts TypeScript +await sdk.runProcess({ + command: "apt-get", + args: ["install", "-y", "dmz-cursor-theme"], +}); + +const xresources = `Xcursor.theme: DMZ-White\nXcursor.size: 24\n`; +await sdk.writeFsFile( + { path: "~/.local/state/sandbox-agent/desktop/home/.Xresources" }, + xresources, +); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/processes/run" \ + -H "Content-Type: application/json" \ + -d '{"command":"apt-get","args":["install","-y","dmz-cursor-theme"]}' + +curl -X PUT "http://127.0.0.1:2468/v1/fs/file?path=~/.local/state/sandbox-agent/desktop/home/.Xresources" \ + -H "Content-Type: application/octet-stream" \ + --data-binary 'Xcursor.theme: DMZ-White\nXcursor.size: 24' +``` + + + +Run `xrdb -merge ~/.Xresources` (via the autostart or process API) after writing the file for changes to take effect. + + +### Shell and terminal + +No terminal emulator or shell is launched by default. Add one to the openbox autostart: + +```sh +# In ~/.config/openbox/autostart +xterm -geometry 120x40+50+50 & +``` + +To use a different shell, set the `SHELL` environment variable in your Dockerfile or install your preferred shell and configure the terminal to use it. + +### GTK theme + +Applications using GTK will pick up settings from `~/.config/gtk-3.0/settings.ini`: + + +```ts TypeScript +const gtkSettings = `[Settings] +gtk-theme-name=Adwaita +gtk-icon-theme-name=Adwaita +gtk-font-name=DejaVu Sans 10 +gtk-cursor-theme-name=DMZ-White +gtk-cursor-theme-size=24 +`; + +await sdk.mkdirFs({ path: "~/.local/state/sandbox-agent/desktop/home/.config/gtk-3.0" }); +await sdk.writeFsFile( + { path: "~/.local/state/sandbox-agent/desktop/home/.config/gtk-3.0/settings.ini" }, + gtkSettings, +); +``` + +```bash cURL +curl -X POST "http://127.0.0.1:2468/v1/fs/mkdir?path=~/.local/state/sandbox-agent/desktop/home/.config/gtk-3.0" + +curl -X PUT "http://127.0.0.1:2468/v1/fs/file?path=~/.local/state/sandbox-agent/desktop/home/.config/gtk-3.0/settings.ini" \ + -H "Content-Type: application/octet-stream" \ + --data-binary @settings.ini +``` + + +### Summary of configuration paths + +All paths are relative to the desktop HOME directory (`~/.local/state/sandbox-agent/desktop/home`). + +| What | Path | Notes | +|------|------|-------| +| Openbox config | `.config/openbox/rc.xml` | Window manager theme, keybindings, behavior | +| Autostart | `.config/openbox/autostart` | Shell script run on desktop start | +| Custom fonts | `.local/share/fonts/` | TTF/OTF files, run `fc-cache -fv` after | +| Cursor theme | `.Xresources` | Requires `xrdb -merge` to apply | +| GTK 3 settings | `.config/gtk-3.0/settings.ini` | Theme, icons, fonts for GTK apps | +| Wallpaper | Any path, referenced from autostart | Requires `feh` or similar tool | diff --git a/docs/docs.json b/docs/docs.json index 16620fe..0c2b19a 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -87,7 +87,7 @@ }, { "group": "System", - "pages": ["file-system", "processes"] + "pages": ["file-system", "processes", "computer-use", "common-software"] }, { "group": "Orchestration", diff --git a/frontend/packages/inspector/src/components/debug/DesktopTab.tsx b/frontend/packages/inspector/src/components/debug/DesktopTab.tsx index 53b3221..1d95d3e 100644 --- a/frontend/packages/inspector/src/components/debug/DesktopTab.tsx +++ b/frontend/packages/inspector/src/components/debug/DesktopTab.tsx @@ -1,33 +1,43 @@ -import { Loader2, Monitor, Play, RefreshCw, Square, Camera } from "lucide-react"; +import { Camera, Circle, Download, Loader2, Monitor, Play, RefreshCw, Square, Trash2, Video } from "lucide-react"; import { useCallback, useEffect, useMemo, useState } from "react"; import { SandboxAgentError } from "sandbox-agent"; -import type { DesktopStatusResponse, SandboxAgent } from "sandbox-agent"; - +import type { DesktopRecordingInfo, DesktopStatusResponse, SandboxAgent } from "sandbox-agent"; +import { DesktopViewer } from "@sandbox-agent/react"; +import type { DesktopViewerClient } from "@sandbox-agent/react"; const MIN_SPIN_MS = 350; - const extractErrorMessage = (error: unknown, fallback: string): string => { if (error instanceof SandboxAgentError && error.problem?.detail) return error.problem.detail; if (error instanceof Error) return error.message; return fallback; }; - const formatStartedAt = (value: string | null | undefined): string => { - if (!value) { - return "Not started"; - } + if (!value) return "Not started"; const parsed = new Date(value); return Number.isNaN(parsed.getTime()) ? value : parsed.toLocaleString(); }; - +const formatBytes = (bytes: number): string => { + if (bytes === 0) return "0 B"; + const units = ["B", "KB", "MB", "GB"]; + const i = Math.floor(Math.log(bytes) / Math.log(1024)); + return `${(bytes / 1024 ** i).toFixed(i > 0 ? 1 : 0)} ${units[i]}`; +}; +const formatDuration = (start: string, end?: string | null): string => { + const startMs = new Date(start).getTime(); + const endMs = end ? new Date(end).getTime() : Date.now(); + if (Number.isNaN(startMs) || Number.isNaN(endMs)) return "Unknown"; + const seconds = Math.round((endMs - startMs) / 1000); + if (seconds < 60) return `${seconds}s`; + const mins = Math.floor(seconds / 60); + const secs = seconds % 60; + return `${mins}m ${secs}s`; +}; const createScreenshotUrl = async (bytes: Uint8Array): Promise => { const payload = new Uint8Array(bytes.byteLength); payload.set(bytes); const blob = new Blob([payload.buffer], { type: "image/png" }); - if (typeof URL.createObjectURL === "function") { return URL.createObjectURL(blob); } - return await new Promise((resolve, reject) => { const reader = new FileReader(); reader.onerror = () => reject(reader.error ?? new Error("Unable to read screenshot blob.")); @@ -41,22 +51,42 @@ const createScreenshotUrl = async (bytes: Uint8Array): Promise => { reader.readAsDataURL(blob); }); }; - const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => { const [status, setStatus] = useState(null); const [loading, setLoading] = useState(false); const [refreshing, setRefreshing] = useState(false); const [acting, setActing] = useState<"start" | "stop" | null>(null); const [error, setError] = useState(null); - const [width, setWidth] = useState("1440"); const [height, setHeight] = useState("900"); const [dpi, setDpi] = useState("96"); - + // Screenshot fallback const [screenshotUrl, setScreenshotUrl] = useState(null); const [screenshotLoading, setScreenshotLoading] = useState(false); const [screenshotError, setScreenshotError] = useState(null); - + // Live view + const [liveViewActive, setLiveViewActive] = useState(false); + const [liveViewError, setLiveViewError] = useState(null); + // Memoize the client as a DesktopViewerClient so the reference is stable + // across renders and doesn't cause the DesktopViewer effect to re-fire. + const viewerClient = useMemo(() => { + const c = getClient(); + return { + startDesktopStream: () => c.startDesktopStream(), + stopDesktopStream: () => c.stopDesktopStream(), + connectDesktopStream: (opts?: Parameters[0]) => c.connectDesktopStream(opts), + }; + }, [getClient]); + // Recording + const [recordings, setRecordings] = useState([]); + const [recordingLoading, setRecordingLoading] = useState(false); + const [recordingActing, setRecordingActing] = useState<"start" | "stop" | null>(null); + const [recordingError, setRecordingError] = useState(null); + const [recordingFps, setRecordingFps] = useState("30"); + const [deletingRecordingId, setDeletingRecordingId] = useState(null); + const [downloadingRecordingId, setDownloadingRecordingId] = useState(null); + // Active recording tracking + const activeRecording = useMemo(() => recordings.find((r) => r.status === "recording"), [recordings]); const revokeScreenshotUrl = useCallback(() => { setScreenshotUrl((current) => { if (current?.startsWith("blob:") && typeof URL.revokeObjectURL === "function") { @@ -65,14 +95,10 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => { return null; }); }, []); - const loadStatus = useCallback( async (mode: "initial" | "refresh" = "initial") => { - if (mode === "initial") { - setLoading(true); - } else { - setRefreshing(true); - } + if (mode === "initial") setLoading(true); + else setRefreshing(true); setError(null); try { const next = await getClient().getDesktopStatus(); @@ -88,7 +114,6 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => { }, [getClient], ); - const refreshScreenshot = useCallback(async () => { setScreenshotLoading(true); setScreenshotError(null); @@ -103,25 +128,38 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => { setScreenshotLoading(false); } }, [getClient, revokeScreenshotUrl]); - + const loadRecordings = useCallback(async () => { + setRecordingLoading(true); + setRecordingError(null); + try { + const result = await getClient().listDesktopRecordings(); + setRecordings(result.recordings); + } catch (loadError) { + setRecordingError(extractErrorMessage(loadError, "Unable to load recordings.")); + } finally { + setRecordingLoading(false); + } + }, [getClient]); useEffect(() => { void loadStatus(); }, [loadStatus]); - useEffect(() => { if (status?.state === "active") { - void refreshScreenshot(); + void loadRecordings(); } else { revokeScreenshotUrl(); + setLiveViewActive(false); } - }, [refreshScreenshot, revokeScreenshotUrl, status?.state]); - + }, [status?.state, loadRecordings, revokeScreenshotUrl]); useEffect(() => { - return () => { - revokeScreenshotUrl(); - }; + return () => revokeScreenshotUrl(); }, [revokeScreenshotUrl]); - + // Poll recording list while a recording is active + useEffect(() => { + if (!activeRecording) return; + const interval = setInterval(() => void loadRecordings(), 3000); + return () => clearInterval(interval); + }, [activeRecording, loadRecordings]); const handleStart = async () => { const parsedWidth = Number.parseInt(width, 10); const parsedHeight = Number.parseInt(height, 10); @@ -136,9 +174,6 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => { dpi: Number.isFinite(parsedDpi) ? parsedDpi : undefined, }); setStatus(next); - if (next.state === "active") { - await refreshScreenshot(); - } } catch (startError) { setError(extractErrorMessage(startError, "Unable to start desktop runtime.")); await loadStatus("refresh"); @@ -150,7 +185,6 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => { setActing(null); } }; - const handleStop = async () => { setActing("stop"); setError(null); @@ -159,6 +193,7 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => { const next = await getClient().stopDesktop(); setStatus(next); revokeScreenshotUrl(); + setLiveViewActive(false); } catch (stopError) { setError(extractErrorMessage(stopError, "Unable to stop desktop runtime.")); await loadStatus("refresh"); @@ -170,15 +205,71 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => { setActing(null); } }; - + const handleStartRecording = async () => { + const fps = Number.parseInt(recordingFps, 10); + setRecordingActing("start"); + setRecordingError(null); + try { + await getClient().startDesktopRecording({ + fps: Number.isFinite(fps) && fps > 0 ? fps : undefined, + }); + await loadRecordings(); + } catch (err) { + setRecordingError(extractErrorMessage(err, "Unable to start recording.")); + } finally { + setRecordingActing(null); + } + }; + const handleStopRecording = async () => { + setRecordingActing("stop"); + setRecordingError(null); + try { + await getClient().stopDesktopRecording(); + await loadRecordings(); + } catch (err) { + setRecordingError(extractErrorMessage(err, "Unable to stop recording.")); + } finally { + setRecordingActing(null); + } + }; + const handleDeleteRecording = async (id: string) => { + setDeletingRecordingId(id); + try { + await getClient().deleteDesktopRecording(id); + setRecordings((prev) => prev.filter((r) => r.id !== id)); + } catch (err) { + setRecordingError(extractErrorMessage(err, "Unable to delete recording.")); + } finally { + setDeletingRecordingId(null); + } + }; + const handleDownloadRecording = async (id: string, fileName: string) => { + setDownloadingRecordingId(id); + try { + const bytes = await getClient().downloadDesktopRecording(id); + const blob = new Blob([bytes], { type: "video/mp4" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = fileName; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + } catch (err) { + setRecordingError(extractErrorMessage(err, "Unable to download recording.")); + } finally { + setDownloadingRecordingId(null); + } + }; const canRefreshScreenshot = status?.state === "active"; + const isActive = status?.state === "active"; const resolutionLabel = useMemo(() => { const resolution = status?.resolution; if (!resolution) return "Unknown"; const dpiLabel = resolution.dpi ? ` @ ${resolution.dpi} DPI` : ""; return `${resolution.width} x ${resolution.height}${dpiLabel}`; }, [status?.resolution]); - return (
@@ -186,15 +277,16 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => { Refresh Status - + {isActive && !liveViewActive && ( + + )}
- {error &&
{error}
} {screenshotError &&
{screenshotError}
} - + {/* ========== Runtime Section ========== */}
@@ -209,7 +301,6 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => { {status?.state ?? "unknown"}
-
Display
@@ -224,7 +315,6 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => {
{formatStartedAt(status?.startedAt)}
-
@@ -239,19 +329,21 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => { setDpi(event.target.value)} inputMode="numeric" />
-
- - + {isActive ? ( + + ) : ( + + )}
- + {/* ========== Missing Dependencies ========== */} {status?.missingDependencies && status.missingDependencies.length > 0 && (
@@ -274,7 +366,179 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => { )}
)} - + {/* ========== Live View Section ========== */} +
+
+ + + {isActive && ( + + )} +
+ {liveViewError && ( +
+ {liveViewError} +
+ )} + {!isActive &&
Start the desktop runtime to enable live view.
} + {isActive && liveViewActive && } + {isActive && !liveViewActive && ( + <> + {screenshotUrl ? ( +
+ Desktop screenshot +
+ ) : ( +
Click "Start Stream" for live desktop view, or use the Screenshot button above.
+ )} + + )} +
+ {/* ========== Recording Section ========== */} +
+
+ + + Recording + + {activeRecording && Recording} +
+ {recordingError && ( +
+ {recordingError} +
+ )} + {!isActive &&
Start the desktop runtime to enable recording.
} + {isActive && ( + <> +
+
+ + setRecordingFps(e.target.value)} + inputMode="numeric" + style={{ maxWidth: 80 }} + disabled={!!activeRecording} + /> +
+
+
+ {!activeRecording ? ( + + ) : ( + + )} + +
+ {recordings.length > 0 && ( +
+ {recordings.map((rec) => ( +
+
+
+ + {rec.fileName} + + + {rec.status} + +
+ {rec.status === "completed" && ( +
+ + +
+ )} +
+
+ {formatBytes(rec.bytes)} + {" \u00b7 "} + {formatDuration(rec.startedAt, rec.endedAt)} + {" \u00b7 "} + {formatStartedAt(rec.startedAt)} +
+
+ ))} +
+ )} + {recordings.length === 0 && !recordingLoading && ( +
+ No recordings yet. Click "Start Recording" to begin. +
+ )} + + )} +
+ {/* ========== Diagnostics Section ========== */} {(status?.lastError || status?.runtimeLogPath || (status?.processes?.length ?? 0) > 0) && (
@@ -314,27 +578,7 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => { )}
)} - -
-
- Latest Screenshot - {status?.state === "active" ? Manual refresh only : null} -
- - {loading ?
Loading...
: null} - {!loading && !screenshotUrl && ( -
- {status?.state === "active" ? "No screenshot loaded yet." : "Start the desktop runtime to capture a screenshot."} -
- )} - {screenshotUrl && ( -
- Desktop screenshot -
- )} -
); }; - export default DesktopTab; diff --git a/frontend/packages/inspector/vite.config.ts b/frontend/packages/inspector/vite.config.ts index 6496813..d398b20 100644 --- a/frontend/packages/inspector/vite.config.ts +++ b/frontend/packages/inspector/vite.config.ts @@ -8,7 +8,7 @@ export default defineConfig(({ command }) => ({ port: 5173, proxy: { "/v1": { - target: "http://localhost:2468", + target: process.env.SANDBOX_AGENT_URL || "http://localhost:2468", changeOrigin: true, ws: true, }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fe82ce0..b2b99c3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5,8 +5,8 @@ settings: excludeLinksFromLockfile: false overrides: - '@types/react': ^18.3.3 - '@types/react-dom': ^18.3.0 + '@types/react': ^19.1.12 + '@types/react-dom': ^19.1.6 importers: @@ -57,7 +57,7 @@ importers: dependencies: '@cloudflare/sandbox': specifier: latest - version: 0.7.17(@opencode-ai/sdk@1.2.24) + version: 0.7.18(@opencode-ai/sdk@1.2.24) hono: specifier: ^4.12.2 version: 4.12.2 @@ -73,16 +73,16 @@ importers: devDependencies: '@cloudflare/workers-types': specifier: latest - version: 4.20260313.1 + version: 4.20260317.1 '@types/node': specifier: latest version: 25.5.0 '@types/react': - specifier: ^18.3.3 - version: 18.3.27 + specifier: ^19.1.12 + version: 19.2.14 '@types/react-dom': - specifier: ^18.3.0 - version: 18.3.7(@types/react@18.3.27) + specifier: ^19.1.6 + version: 19.2.3(@types/react@19.2.14) '@vitejs/plugin-react': specifier: ^4.5.0 version: 4.7.0(vite@6.4.1(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2)) @@ -97,7 +97,7 @@ importers: version: 3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) wrangler: specifier: latest - version: 4.73.0(@cloudflare/workers-types@4.20260313.1) + version: 4.74.0(@cloudflare/workers-types@4.20260317.1) examples/computesdk: dependencies: @@ -106,7 +106,7 @@ importers: version: link:../shared computesdk: specifier: latest - version: 2.5.0 + version: 2.5.1 sandbox-agent: specifier: workspace:* version: link:../../sdks/typescript @@ -154,13 +154,13 @@ importers: dockerode: specifier: latest version: 4.0.9 + get-port: + specifier: latest + version: 7.1.0 sandbox-agent: specifier: workspace:* version: link:../../sdks/typescript devDependencies: - '@types/dockerode': - specifier: latest - version: 4.0.1 '@types/node': specifier: latest version: 25.5.0 @@ -345,9 +345,6 @@ importers: '@sandbox-agent/example-shared': specifier: workspace:* version: link:../shared - '@sandbox-agent/persist-postgres': - specifier: workspace:* - version: link:../../sdks/persist-postgres pg: specifier: latest version: 8.20.0 @@ -373,13 +370,16 @@ importers: '@sandbox-agent/example-shared': specifier: workspace:* version: link:../shared - '@sandbox-agent/persist-sqlite': - specifier: workspace:* - version: link:../../sdks/persist-sqlite + better-sqlite3: + specifier: ^11.0.0 + version: 11.10.0 sandbox-agent: specifier: workspace:* version: link:../../sdks/typescript devDependencies: + '@types/better-sqlite3': + specifier: ^7.0.0 + version: 7.6.13 '@types/node': specifier: latest version: 25.5.0 @@ -492,12 +492,9 @@ importers: '@sandbox-agent/foundry-shared': specifier: workspace:* version: link:../shared - '@sandbox-agent/persist-rivet': - specifier: workspace:* - version: link:../../../sdks/persist-rivet better-auth: specifier: ^1.5.5 - version: 1.5.5(@cloudflare/workers-types@4.20260313.1)(drizzle-kit@0.31.9)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260313.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0))(pg@8.20.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(solid-js@1.9.11)(vitest@3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2)) + version: 1.5.5(@cloudflare/workers-types@4.20260317.1)(drizzle-kit@0.31.9)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260317.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0))(pg@8.20.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(solid-js@1.9.11)(vitest@3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2)) dockerode: specifier: ^4.0.9 version: 4.0.9 @@ -506,7 +503,7 @@ importers: version: 0.31.9 drizzle-orm: specifier: ^0.44.5 - version: 0.44.7(@cloudflare/workers-types@4.20260313.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(better-sqlite3@11.10.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0) + version: 0.44.7(@cloudflare/workers-types@4.20260317.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(better-sqlite3@11.10.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0) hono: specifier: ^4.11.9 version: 4.12.2 @@ -515,7 +512,7 @@ importers: version: 10.3.1 rivetkit: specifier: https://pkg.pr.new/rivet-dev/rivet/rivetkit@791500a - version: https://pkg.pr.new/rivet-dev/rivet/rivetkit@791500a(@e2b/code-interpreter@2.3.3)(@hono/node-server@1.19.9(hono@4.12.2))(@hono/node-ws@1.3.0(@hono/node-server@1.19.9(hono@4.12.2))(hono@4.12.2))(@standard-schema/spec@1.1.0)(dockerode@4.0.9)(drizzle-kit@0.31.9)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260313.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0))(ws@8.19.0) + version: https://pkg.pr.new/rivet-dev/rivet/rivetkit@791500a(@e2b/code-interpreter@2.3.3)(@hono/node-server@1.19.9(hono@4.12.2))(@hono/node-ws@1.3.0(@hono/node-server@1.19.9(hono@4.12.2))(hono@4.12.2))(@standard-schema/spec@1.1.0)(dockerode@4.0.9)(drizzle-kit@0.31.9)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260317.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0))(ws@8.19.0) sandbox-agent: specifier: workspace:* version: link:../../../sdks/typescript @@ -546,14 +543,14 @@ importers: version: 19.2.4 rivetkit: specifier: 2.1.6 - version: 2.1.6(@hono/node-server@1.19.9(hono@4.12.2))(@hono/node-ws@1.3.0(@hono/node-server@1.19.9(hono@4.12.2))(hono@4.12.2))(@standard-schema/spec@1.1.0)(drizzle-kit@0.31.9)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260313.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(better-sqlite3@11.10.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0))(ws@8.19.0) + version: 2.1.6(@hono/node-server@1.19.9(hono@4.12.2))(@hono/node-ws@1.3.0(@hono/node-server@1.19.9(hono@4.12.2))(hono@4.12.2))(@standard-schema/spec@1.1.0)(drizzle-kit@0.31.9)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260317.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(better-sqlite3@11.10.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0))(ws@8.19.0) sandbox-agent: specifier: workspace:* version: link:../../../sdks/typescript devDependencies: '@types/react': - specifier: ^18.3.3 - version: 18.3.27 + specifier: ^19.1.12 + version: 19.2.14 tsup: specifier: ^8.5.0 version: 8.5.1(jiti@1.21.7)(postcss@8.5.6)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.2) @@ -580,7 +577,7 @@ importers: version: 3.13.22(react-dom@19.2.4(react@19.2.4))(react@19.2.4) baseui: specifier: ^16.1.1 - version: 16.1.1(@types/react@18.3.27)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(styletron-react@6.1.1(react@19.2.4)) + version: 16.1.1(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(styletron-react@6.1.1(react@19.2.4)) lucide-react: specifier: ^0.542.0 version: 0.542.0(react@19.2.4) @@ -602,19 +599,19 @@ importers: devDependencies: '@react-grab/mcp': specifier: ^0.1.13 - version: 0.1.27(@types/react@18.3.27)(react@19.2.4) + version: 0.1.27(@types/react@19.2.14)(react@19.2.4) '@types/react': - specifier: ^18.3.3 - version: 18.3.27 + specifier: ^19.1.12 + version: 19.2.14 '@types/react-dom': - specifier: ^18.3.0 - version: 18.3.7(@types/react@18.3.27) + specifier: ^19.1.6 + version: 19.2.3(@types/react@19.2.14) '@vitejs/plugin-react': specifier: ^5.0.3 version: 5.1.4(vite@7.3.1(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2)) react-grab: specifier: ^0.1.13 - version: 0.1.27(@types/react@18.3.27)(react@19.2.4) + version: 0.1.27(@types/react@19.2.14)(react@19.2.4) tsup: specifier: ^8.5.0 version: 8.5.1(jiti@1.21.7)(postcss@8.5.6)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.2) @@ -640,9 +637,6 @@ importers: frontend/packages/inspector: dependencies: - '@sandbox-agent/persist-indexeddb': - specifier: workspace:* - version: link:../../../sdks/persist-indexeddb lucide-react: specifier: ^0.469.0 version: 0.469.0(react@18.3.1) @@ -657,11 +651,11 @@ importers: specifier: workspace:* version: link:../../../sdks/react '@types/react': - specifier: ^18.3.3 - version: 18.3.27 + specifier: ^19.1.12 + version: 19.2.14 '@types/react-dom': - specifier: ^18.3.0 - version: 18.3.7(@types/react@18.3.27) + specifier: ^19.1.6 + version: 19.2.3(@types/react@19.2.14) '@vitejs/plugin-react': specifier: ^4.3.1 version: 4.7.0(vite@5.4.21(@types/node@25.5.0)) @@ -688,7 +682,7 @@ importers: dependencies: '@astrojs/react': specifier: ^4.2.0 - version: 4.4.2(@types/node@25.5.0)(@types/react-dom@18.3.7(@types/react@18.3.27))(@types/react@18.3.27)(jiti@1.21.7)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(tsx@4.21.0)(yaml@2.8.2) + version: 4.4.2(@types/node@25.5.0)(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(jiti@1.21.7)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(tsx@4.21.0)(yaml@2.8.2) '@astrojs/sitemap': specifier: ^3.2.0 version: 3.7.0 @@ -715,11 +709,11 @@ importers: version: 3.4.19(tsx@4.21.0)(yaml@2.8.2) devDependencies: '@types/react': - specifier: ^18.3.3 - version: 18.3.27 + specifier: ^19.1.12 + version: 19.2.14 '@types/react-dom': - specifier: ^18.3.0 - version: 18.3.7(@types/react@18.3.27) + specifier: ^19.1.6 + version: 19.2.3(@types/react@19.2.14) typescript: specifier: ^5.7.0 version: 5.9.3 @@ -799,8 +793,8 @@ importers: sdks/acp-http-client: dependencies: '@agentclientprotocol/sdk': - specifier: ^0.14.1 - version: 0.14.1(zod@4.3.6) + specifier: ^0.16.1 + version: 0.16.1(zod@4.3.6) devDependencies: '@types/node': specifier: ^22.0.0 @@ -900,57 +894,30 @@ importers: sdks/gigacode/platforms/win32-x64: {} sdks/persist-indexeddb: - dependencies: - sandbox-agent: - specifier: workspace:* - version: link:../typescript devDependencies: '@types/node': specifier: ^22.0.0 version: 22.19.7 - fake-indexeddb: - specifier: ^6.2.4 - version: 6.2.5 tsup: specifier: ^8.0.0 version: 8.5.1(jiti@1.21.7)(postcss@8.5.6)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.2) typescript: specifier: ^5.7.0 version: 5.9.3 - vitest: - specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) sdks/persist-postgres: - dependencies: - pg: - specifier: ^8.16.3 - version: 8.18.0 - sandbox-agent: - specifier: workspace:* - version: link:../typescript devDependencies: '@types/node': specifier: ^22.0.0 version: 22.19.7 - '@types/pg': - specifier: ^8.15.6 - version: 8.16.0 tsup: specifier: ^8.0.0 version: 8.5.1(jiti@1.21.7)(postcss@8.5.6)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.2) typescript: specifier: ^5.7.0 version: 5.9.3 - vitest: - specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) sdks/persist-rivet: - dependencies: - sandbox-agent: - specifier: workspace:* - version: link:../typescript devDependencies: '@types/node': specifier: ^22.0.0 @@ -961,22 +928,9 @@ importers: typescript: specifier: ^5.7.0 version: 5.9.3 - vitest: - specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) sdks/persist-sqlite: - dependencies: - better-sqlite3: - specifier: ^11.0.0 - version: 11.10.0 - sandbox-agent: - specifier: workspace:* - version: link:../typescript devDependencies: - '@types/better-sqlite3': - specifier: ^7.0.0 - version: 7.6.13 '@types/node': specifier: ^22.0.0 version: 22.19.7 @@ -986,25 +940,22 @@ importers: typescript: specifier: ^5.7.0 version: 5.9.3 - vitest: - specifier: ^3.0.0 - version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.7)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2) sdks/react: dependencies: '@tanstack/react-virtual': specifier: ^3.13.22 - version: 3.13.22(react-dom@19.2.4(react@18.3.1))(react@18.3.1) + version: 3.13.22(react-dom@19.2.4(react@19.2.4))(react@19.2.4) ghostty-web: specifier: ^0.4.0 version: 0.4.0 devDependencies: '@types/react': - specifier: ^18.3.3 - version: 18.3.27 + specifier: ^19.1.12 + version: 19.2.14 react: - specifier: ^18.3.1 - version: 18.3.1 + specifier: ^19.1.1 + version: 19.2.4 sandbox-agent: specifier: workspace:* version: link:../typescript @@ -1028,12 +979,39 @@ importers: specifier: workspace:* version: link:../cli devDependencies: + '@cloudflare/sandbox': + specifier: '>=0.1.0' + version: 0.7.18(@opencode-ai/sdk@1.2.24) + '@daytonaio/sdk': + specifier: '>=0.12.0' + version: 0.151.0(ws@8.19.0) + '@e2b/code-interpreter': + specifier: '>=1.0.0' + version: 2.3.3 + '@types/dockerode': + specifier: ^4.0.0 + version: 4.0.1 '@types/node': specifier: ^22.0.0 version: 22.19.7 '@types/ws': specifier: ^8.18.1 version: 8.18.1 + '@vercel/sandbox': + specifier: '>=0.1.0' + version: 1.8.1 + computesdk: + specifier: '>=0.1.0' + version: 2.5.1 + dockerode: + specifier: '>=4.0.0' + version: 4.0.9 + get-port: + specifier: '>=7.0.0' + version: 7.1.0 + modal: + specifier: '>=0.1.0' + version: 0.7.3 openapi-typescript: specifier: ^6.7.0 version: 6.7.6 @@ -1073,6 +1051,11 @@ packages: peerDependencies: zod: ^3.25.0 || ^4.0.0 + '@agentclientprotocol/sdk@0.16.1': + resolution: {integrity: sha512-1ad+Sc/0sCtZGHthxxvgEUo5Wsbw16I+aF+YwdiLnPwkZG8KAGUEAPK6LM6Pf69lCyJPt1Aomk1d+8oE3C4ZEw==} + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + '@alloc/quick-lru@5.2.0': resolution: {integrity: sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==} engines: {node: '>=10'} @@ -1106,8 +1089,8 @@ packages: resolution: {integrity: sha512-1tl95bpGfuaDMDn8O3x/5Dxii1HPvzjvpL2YTuqOOrQehs60I2DKiDgh1jrKc7G8lv+LQT5H15V6QONQ+9waeQ==} engines: {node: 18.20.8 || ^20.3.0 || >=22.0.0} peerDependencies: - '@types/react': ^18.3.3 - '@types/react-dom': ^18.3.0 + '@types/react': ^19.1.12 + '@types/react-dom': ^19.1.6 react: ^17.0.2 || ^18.0.0 || ^19.0.0 react-dom: ^17.0.2 || ^18.0.0 || ^19.0.0 @@ -1614,8 +1597,8 @@ packages: resolution: {integrity: sha512-SIOD2DxrRRwQ+jgzlXCqoEFiKOFqaPjhnNTGKXSRLvp1HiOvapLaFG2kEr9dYQTYe8rKrd9uvDUzmAITeNyaHQ==} engines: {node: '>=18.0.0'} - '@cloudflare/sandbox@0.7.17': - resolution: {integrity: sha512-dS6IdIq8OHg7jg08wz18BtP6TkBSaVrixq+kN4Zm5/6N9Vir+PbfebEFL23lM5UcaPoKx6Gttnhby8e79PSMOA==} + '@cloudflare/sandbox@0.7.18': + resolution: {integrity: sha512-v9hNXik88TfhLu83RRf1I8t98L86bt2RJVJB+EPIyZB7vBVwsx8w/veQ1BoJfbf/MSETtc/zLpQROgtbe5RYRw==} peerDependencies: '@openai/agents': ^0.3.3 '@opencode-ai/sdk': ^1.1.40 @@ -1667,8 +1650,8 @@ packages: cpu: [x64] os: [win32] - '@cloudflare/workers-types@4.20260313.1': - resolution: {integrity: sha512-jMEeX3RKfOSVqqXRKr/ulgglcTloeMzSH3FdzIfqJHtvc12/ELKd5Ldsg8ZHahKX/4eRxYdw3kbzb8jLXbq/jQ==} + '@cloudflare/workers-types@4.20260317.1': + resolution: {integrity: sha512-+G4eVwyCpm8Au1ex8vQBCuA9wnwqetz4tPNRoB/53qvktERWBRMQnrtvC1k584yRE3emMThtuY0gWshvSJ++PQ==} '@computesdk/cmd@0.4.1': resolution: {integrity: sha512-hhcYrwMnOpRSwWma3gkUeAVsDFG56nURwSaQx8vCepv0IuUv39bK4mMkgszolnUQrVjBDdW7b3lV+l5B2S8fRA==} @@ -3636,27 +3619,21 @@ packages: '@types/node@25.5.0': resolution: {integrity: sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw==} - '@types/pg@8.16.0': - resolution: {integrity: sha512-RmhMd/wD+CF8Dfo+cVIy3RR5cl8CyfXQ0tGgW6XBL8L4LM/UTEbNXYRbLwU6w+CgrKBNbrQWt4FUtTfaU5jSYQ==} - '@types/pg@8.18.0': resolution: {integrity: sha512-gT+oueVQkqnj6ajGJXblFR4iavIXWsGAFCk3dP4Kki5+a9R4NMt0JARdk6s8cUKcfUoqP5dAtDSLU8xYUTFV+Q==} - '@types/prop-types@15.7.15': - resolution: {integrity: sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==} - - '@types/react-dom@18.3.7': - resolution: {integrity: sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==} + '@types/react-dom@19.2.3': + resolution: {integrity: sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==} peerDependencies: - '@types/react': ^18.3.3 + '@types/react': ^19.1.12 '@types/react-reconciler@0.28.9': resolution: {integrity: sha512-HHM3nxyUZ3zAylX8ZEyrDNd2XZOnQ0D5XfunJF5FLQnZbHHYq4UWvW1QfelQNXv1ICNkwYhfxjwfnqivYB6bFg==} peerDependencies: - '@types/react': ^18.3.3 + '@types/react': ^19.1.12 - '@types/react@18.3.27': - resolution: {integrity: sha512-cisd7gxkzjBKU2GgdYrTdtQx1SORymWyaAFhaxQPK9bYO9ot3Y5OikQRvY0VYQtvwjeQnizCINJAenh/V7MK2w==} + '@types/react@19.2.14': + resolution: {integrity: sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==} '@types/retry@0.12.2': resolution: {integrity: sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==} @@ -4192,8 +4169,8 @@ packages: compare-versions@6.1.1: resolution: {integrity: sha512-4hm4VPpIecmlg59CHXnRDnqGplJFrbLG4aFEl5vl6cK1u76ws3LLvX7ikFnTDl5vo39sjWD6AaDPYodJp/NNHg==} - computesdk@2.5.0: - resolution: {integrity: sha512-HDOSw1D6ogR7wwlBtr5ZZtRFhfNxdPBW1omLaApxDwAvdkBW4PGoIfYyMPX0N1LpaDl5NWwWALMvmtW2p4dafg==} + computesdk@2.5.1: + resolution: {integrity: sha512-zDzgdztvloTEm7cfVxSYX+kaQXlg+2UQBzmsisH61LTVcEPaZA7CCI/T8eC/eCxGzOjUjs5n68xo/ymK3w7Ebg==} confbox@0.1.8: resolution: {integrity: sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==} @@ -5623,8 +5600,8 @@ packages: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} - miniflare@4.20260312.0: - resolution: {integrity: sha512-pieP2rfXynPT6VRINYaiHe/tfMJ4c5OIhqRlIdLF6iZ9g5xgpEmvimvIgMpgAdDJuFlrLcwDUi8MfAo2R6dt/w==} + miniflare@4.20260312.1: + resolution: {integrity: sha512-YSWxec9ssisqkQgaCgcIQxZlB41E9hMiq1nxUgxXHRrE9NsfyC6ptSt8yfgBobsKIseAVKLTB/iEDpMumBv8oA==} engines: {node: '>=18.0.0'} hasBin: true @@ -5894,9 +5871,6 @@ packages: pg-cloudflare@1.3.0: resolution: {integrity: sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==} - pg-connection-string@2.11.0: - resolution: {integrity: sha512-kecgoJwhOpxYU21rZjULrmrBJ698U2RxXofKVzOn5UDj61BPj/qMb7diYUR1nLScCDbrztQFl1TaQZT0t1EtzQ==} - pg-connection-string@2.12.0: resolution: {integrity: sha512-U7qg+bpswf3Cs5xLzRqbXbQl85ng0mfSV/J0nnA31MCLgvEaAo7CIhmeyrmJpOr7o+zm0rXK+hNnT5l9RHkCkQ==} @@ -5904,11 +5878,6 @@ packages: resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} engines: {node: '>=4.0.0'} - pg-pool@3.11.0: - resolution: {integrity: sha512-MJYfvHwtGp870aeusDh+hg9apvOe2zmpZJpyt+BMtzUWlVqbhFmMK6bOBXLBUPd7iRtIF9fZplDc7KrPN3PN7w==} - peerDependencies: - pg: '>=8.0' - pg-pool@3.13.0: resolution: {integrity: sha512-gB+R+Xud1gLFuRD/QgOIgGOBE2KCQPaPwkzBBGC9oG69pHTkhQeIuejVIk3/cnDyX39av2AxomQiyPT13WKHQA==} peerDependencies: @@ -5924,15 +5893,6 @@ packages: resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} engines: {node: '>=4'} - pg@8.18.0: - resolution: {integrity: sha512-xqrUDL1b9MbkydY/s+VZ6v+xiMUmOUk7SS9d/1kpyQxoJ6U9AO1oIJyUWVZojbfe5Cc/oluutcgFG4L9RDP1iQ==} - engines: {node: '>= 16.0.0'} - peerDependencies: - pg-native: '>=3.0.1' - peerDependenciesMeta: - pg-native: - optional: true - pg@8.20.0: resolution: {integrity: sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==} engines: {node: '>= 16.0.0'} @@ -6193,7 +6153,7 @@ packages: react-focus-lock@2.13.7: resolution: {integrity: sha512-20lpZHEQrXPb+pp1tzd4ULL6DyO5D2KnR0G69tTDdydrmNhU7pdFmbQUYVyHUgp+xN29IuFR0PVuhOmvaZL9Og==} peerDependencies: - '@types/react': ^18.3.3 + '@types/react': ^19.1.12 react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc peerDependenciesMeta: '@types/react': @@ -6262,7 +6222,7 @@ packages: resolution: {integrity: sha512-tsPZ77GR0pISGYmpCLHAbZTabKXZ7zBniKPVqVMMfnXFyo39zq5g/psIlD5vLTKkjQEhWOO8JhqcHnxkwNu6eA==} engines: {node: '>=8.5.0'} peerDependencies: - '@types/react': ^18.3.3 + '@types/react': ^19.1.12 react: ^16.8.0 peerDependenciesMeta: '@types/react': @@ -7122,7 +7082,7 @@ packages: resolution: {integrity: sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==} engines: {node: '>=10'} peerDependencies: - '@types/react': ^18.3.3 + '@types/react': ^19.1.12 react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc peerDependenciesMeta: '@types/react': @@ -7132,7 +7092,7 @@ packages: resolution: {integrity: sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ==} engines: {node: '>=10'} peerDependencies: - '@types/react': ^18.3.3 + '@types/react': ^19.1.12 react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc peerDependenciesMeta: '@types/react': @@ -7394,8 +7354,8 @@ packages: engines: {node: '>=16'} hasBin: true - wrangler@4.73.0: - resolution: {integrity: sha512-VJXsqKDFCp6OtFEHXITSOR5kh95JOknwPY8m7RyQuWJQguSybJy43m4vhoCSt42prutTef7eeuw7L4V4xiynGw==} + wrangler@4.74.0: + resolution: {integrity: sha512-3qprbhgdUyqYGHZ+Y1k0gsyHLMOlLrKL/HU0LDqLlCkbsKPprUA0/ThE4IZsxD84xAAXY6pv5JUuxS2+OnMa3A==} engines: {node: '>=20.0.0'} hasBin: true peerDependencies: @@ -7536,6 +7496,10 @@ snapshots: dependencies: zod: 4.3.6 + '@agentclientprotocol/sdk@0.16.1(zod@4.3.6)': + dependencies: + zod: 4.3.6 + '@alloc/quick-lru@5.2.0': {} '@antfu/ni@0.23.2': {} @@ -7587,10 +7551,10 @@ snapshots: dependencies: prismjs: 1.30.0 - '@astrojs/react@4.4.2(@types/node@25.5.0)(@types/react-dom@18.3.7(@types/react@18.3.27))(@types/react@18.3.27)(jiti@1.21.7)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(tsx@4.21.0)(yaml@2.8.2)': + '@astrojs/react@4.4.2(@types/node@25.5.0)(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(jiti@1.21.7)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(tsx@4.21.0)(yaml@2.8.2)': dependencies: - '@types/react': 18.3.27 - '@types/react-dom': 18.3.7(@types/react@18.3.27) + '@types/react': 19.2.14 + '@types/react-dom': 19.2.3(@types/react@19.2.14) '@vitejs/plugin-react': 4.7.0(vite@6.4.1(@types/node@25.5.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2)) react: 19.2.4 react-dom: 19.2.4(react@19.2.4) @@ -8372,7 +8336,7 @@ snapshots: '@balena/dockerignore@1.0.2': {} - '@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1)': + '@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1)': dependencies: '@better-auth/utils': 0.3.1 '@better-fetch/fetch': 1.1.21 @@ -8383,39 +8347,39 @@ snapshots: nanostores: 1.1.1 zod: 4.3.6 optionalDependencies: - '@cloudflare/workers-types': 4.20260313.1 + '@cloudflare/workers-types': 4.20260317.1 - '@better-auth/drizzle-adapter@1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260313.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0))': + '@better-auth/drizzle-adapter@1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260317.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0))': dependencies: - '@better-auth/core': 1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1) + '@better-auth/core': 1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1) '@better-auth/utils': 0.3.1 optionalDependencies: - drizzle-orm: 0.44.7(@cloudflare/workers-types@4.20260313.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(better-sqlite3@11.10.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0) + drizzle-orm: 0.44.7(@cloudflare/workers-types@4.20260317.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(better-sqlite3@11.10.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0) - '@better-auth/kysely-adapter@1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(kysely@0.28.11)': + '@better-auth/kysely-adapter@1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(kysely@0.28.11)': dependencies: - '@better-auth/core': 1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1) + '@better-auth/core': 1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1) '@better-auth/utils': 0.3.1 kysely: 0.28.11 - '@better-auth/memory-adapter@1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1)': + '@better-auth/memory-adapter@1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1)': dependencies: - '@better-auth/core': 1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1) + '@better-auth/core': 1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1) '@better-auth/utils': 0.3.1 - '@better-auth/mongo-adapter@1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1)': + '@better-auth/mongo-adapter@1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1)': dependencies: - '@better-auth/core': 1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1) + '@better-auth/core': 1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1) '@better-auth/utils': 0.3.1 - '@better-auth/prisma-adapter@1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1)': + '@better-auth/prisma-adapter@1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1)': dependencies: - '@better-auth/core': 1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1) + '@better-auth/core': 1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1) '@better-auth/utils': 0.3.1 - '@better-auth/telemetry@1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))': + '@better-auth/telemetry@1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))': dependencies: - '@better-auth/core': 1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1) + '@better-auth/core': 1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1) '@better-auth/utils': 0.3.1 '@better-fetch/fetch': 1.1.21 @@ -8497,7 +8461,7 @@ snapshots: '@cloudflare/kv-asset-handler@0.4.2': {} - '@cloudflare/sandbox@0.7.17(@opencode-ai/sdk@1.2.24)': + '@cloudflare/sandbox@0.7.18(@opencode-ai/sdk@1.2.24)': dependencies: '@cloudflare/containers': 0.1.1 aws4fetch: 1.0.20 @@ -8525,7 +8489,7 @@ snapshots: '@cloudflare/workerd-windows-64@1.20260312.1': optional: true - '@cloudflare/workers-types@4.20260313.1': {} + '@cloudflare/workers-types@4.20260317.1': {} '@computesdk/cmd@0.4.1': {} @@ -9610,11 +9574,11 @@ snapshots: prompts: 2.4.2 smol-toml: 1.6.0 - '@react-grab/mcp@0.1.27(@types/react@18.3.27)(react@19.2.4)': + '@react-grab/mcp@0.1.27(@types/react@19.2.14)(react@19.2.4)': dependencies: '@modelcontextprotocol/sdk': 1.27.1(zod@3.25.76) fkill: 9.0.0 - react-grab: 0.1.27(@types/react@18.3.27)(react@19.2.4) + react-grab: 0.1.27(@types/react@19.2.14)(react@19.2.4) zod: 3.25.76 transitivePeerDependencies: - '@cfworker/json-schema' @@ -10227,12 +10191,6 @@ snapshots: react-dom: 19.2.4(react@19.2.4) use-sync-external-store: 1.6.0(react@19.2.4) - '@tanstack/react-virtual@3.13.22(react-dom@19.2.4(react@18.3.1))(react@18.3.1)': - dependencies: - '@tanstack/virtual-core': 3.13.22 - react: 18.3.1 - react-dom: 19.2.4(react@18.3.1) - '@tanstack/react-virtual@3.13.22(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': dependencies: '@tanstack/virtual-core': 3.13.22 @@ -10342,31 +10300,22 @@ snapshots: dependencies: undici-types: 7.18.2 - '@types/pg@8.16.0': - dependencies: - '@types/node': 24.10.9 - pg-protocol: 1.11.0 - pg-types: 2.2.0 - '@types/pg@8.18.0': dependencies: '@types/node': 24.10.9 pg-protocol: 1.11.0 pg-types: 2.2.0 - '@types/prop-types@15.7.15': {} - - '@types/react-dom@18.3.7(@types/react@18.3.27)': + '@types/react-dom@19.2.3(@types/react@19.2.14)': dependencies: - '@types/react': 18.3.27 + '@types/react': 19.2.14 - '@types/react-reconciler@0.28.9(@types/react@18.3.27)': + '@types/react-reconciler@0.28.9(@types/react@19.2.14)': dependencies: - '@types/react': 18.3.27 + '@types/react': 19.2.14 - '@types/react@18.3.27': + '@types/react@19.2.14': dependencies: - '@types/prop-types': 15.7.15 csstype: 3.2.3 '@types/retry@0.12.2': {} @@ -10725,7 +10674,7 @@ snapshots: baseline-browser-mapping@2.9.18: {} - baseui@16.1.1(@types/react@18.3.27)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(styletron-react@6.1.1(react@19.2.4)): + baseui@16.1.1(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(styletron-react@6.1.1(react@19.2.4)): dependencies: '@date-io/date-fns': 2.17.0(date-fns@2.30.0) '@date-io/moment': 2.17.0(moment@2.30.1) @@ -10745,7 +10694,7 @@ snapshots: react: 19.2.4 react-dom: 19.2.4(react@19.2.4) react-dropzone: 9.0.0(react@19.2.4) - react-focus-lock: 2.13.7(@types/react@18.3.27)(react@19.2.4) + react-focus-lock: 2.13.7(@types/react@19.2.14)(react@19.2.4) react-hook-form: 7.71.2(react@19.2.4) react-input-mask: 2.0.4(react-dom@19.2.4(react@19.2.4))(react@19.2.4) react-is: 17.0.2 @@ -10753,7 +10702,7 @@ snapshots: react-movable: 3.4.1(react-dom@19.2.4(react@19.2.4))(react@19.2.4) react-multi-ref: 1.0.2 react-range: 1.10.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - react-uid: 2.3.0(@types/react@18.3.27)(react@19.2.4) + react-uid: 2.3.0(@types/react@19.2.14)(react@19.2.4) react-virtualized: 9.22.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4) react-virtualized-auto-sizer: 1.0.2(react-dom@19.2.4(react@19.2.4))(react@19.2.4) react-window: 1.8.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -10767,15 +10716,15 @@ snapshots: dependencies: tweetnacl: 0.14.5 - better-auth@1.5.5(@cloudflare/workers-types@4.20260313.1)(drizzle-kit@0.31.9)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260313.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0))(pg@8.20.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(solid-js@1.9.11)(vitest@3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2)): + better-auth@1.5.5(@cloudflare/workers-types@4.20260317.1)(drizzle-kit@0.31.9)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260317.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0))(pg@8.20.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(solid-js@1.9.11)(vitest@3.2.4(@types/debug@4.1.12)(@types/node@25.5.0)(jiti@1.21.7)(jsdom@26.1.0)(tsx@4.21.0)(yaml@2.8.2)): dependencies: - '@better-auth/core': 1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1) - '@better-auth/drizzle-adapter': 1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260313.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0)) - '@better-auth/kysely-adapter': 1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(kysely@0.28.11) - '@better-auth/memory-adapter': 1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1) - '@better-auth/mongo-adapter': 1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1) - '@better-auth/prisma-adapter': 1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1) - '@better-auth/telemetry': 1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260313.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1)) + '@better-auth/core': 1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1) + '@better-auth/drizzle-adapter': 1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260317.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0)) + '@better-auth/kysely-adapter': 1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1)(kysely@0.28.11) + '@better-auth/memory-adapter': 1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1) + '@better-auth/mongo-adapter': 1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1) + '@better-auth/prisma-adapter': 1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1))(@better-auth/utils@0.3.1) + '@better-auth/telemetry': 1.5.5(@better-auth/core@1.5.5(@better-auth/utils@0.3.1)(@better-fetch/fetch@1.1.21)(@cloudflare/workers-types@4.20260317.1)(better-call@1.3.2(zod@4.3.6))(jose@6.1.3)(kysely@0.28.11)(nanostores@1.1.1)) '@better-auth/utils': 0.3.1 '@better-fetch/fetch': 1.1.21 '@noble/ciphers': 2.1.1 @@ -10788,7 +10737,7 @@ snapshots: zod: 4.3.6 optionalDependencies: drizzle-kit: 0.31.9 - drizzle-orm: 0.44.7(@cloudflare/workers-types@4.20260313.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(better-sqlite3@11.10.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0) + drizzle-orm: 0.44.7(@cloudflare/workers-types@4.20260317.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(better-sqlite3@11.10.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0) pg: 8.20.0 react: 19.2.4 react-dom: 19.2.4(react@19.2.4) @@ -10817,9 +10766,9 @@ snapshots: dependencies: file-uri-to-path: 1.0.0 - bippy@0.5.32(@types/react@18.3.27)(react@19.2.4): + bippy@0.5.32(@types/react@19.2.14)(react@19.2.4): dependencies: - '@types/react-reconciler': 0.28.9(@types/react@18.3.27) + '@types/react-reconciler': 0.28.9(@types/react@19.2.14) react: 19.2.4 transitivePeerDependencies: - '@types/react' @@ -11041,7 +10990,7 @@ snapshots: compare-versions@6.1.1: {} - computesdk@2.5.0: + computesdk@2.5.1: dependencies: '@computesdk/cmd': 0.4.1 @@ -11409,9 +11358,9 @@ snapshots: transitivePeerDependencies: - supports-color - drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260313.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(better-sqlite3@11.10.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0): + drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260317.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(better-sqlite3@11.10.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0): optionalDependencies: - '@cloudflare/workers-types': 4.20260313.1 + '@cloudflare/workers-types': 4.20260317.1 '@opentelemetry/api': 1.9.0 '@types/better-sqlite3': 7.6.13 '@types/pg': 8.18.0 @@ -11439,7 +11388,7 @@ snapshots: glob: 11.1.0 openapi-fetch: 0.14.1 platform: 1.3.6 - tar: 7.5.6 + tar: 7.5.7 earcut@2.2.4: {} @@ -12724,7 +12673,7 @@ snapshots: mimic-response@3.1.0: {} - miniflare@4.20260312.0: + miniflare@4.20260312.1: dependencies: '@cspotcode/source-map-support': 0.8.1 sharp: 0.34.5 @@ -12998,16 +12947,10 @@ snapshots: pg-cloudflare@1.3.0: optional: true - pg-connection-string@2.11.0: {} - pg-connection-string@2.12.0: {} pg-int8@1.0.1: {} - pg-pool@3.11.0(pg@8.18.0): - dependencies: - pg: 8.18.0 - pg-pool@3.13.0(pg@8.20.0): dependencies: pg: 8.20.0 @@ -13024,16 +12967,6 @@ snapshots: postgres-date: 1.0.7 postgres-interval: 1.2.0 - pg@8.18.0: - dependencies: - pg-connection-string: 2.11.0 - pg-pool: 3.11.0(pg@8.18.0) - pg-protocol: 1.11.0 - pg-types: 2.2.0 - pgpass: 1.0.5 - optionalDependencies: - pg-cloudflare: 1.3.0 - pg@8.20.0: dependencies: pg-connection-string: 2.12.0 @@ -13290,11 +13223,6 @@ snapshots: react: 18.3.1 scheduler: 0.23.2 - react-dom@19.2.4(react@18.3.1): - dependencies: - react: 18.3.1 - scheduler: 0.27.0 - react-dom@19.2.4(react@19.2.4): dependencies: react: 19.2.4 @@ -13308,23 +13236,23 @@ snapshots: prop-types-extra: 1.1.1(react@19.2.4) react: 19.2.4 - react-focus-lock@2.13.7(@types/react@18.3.27)(react@19.2.4): + react-focus-lock@2.13.7(@types/react@19.2.14)(react@19.2.4): dependencies: '@babel/runtime': 7.28.6 focus-lock: 1.3.6 prop-types: 15.8.1 react: 19.2.4 react-clientside-effect: 1.2.8(react@19.2.4) - use-callback-ref: 1.3.3(@types/react@18.3.27)(react@19.2.4) - use-sidecar: 1.1.3(@types/react@18.3.27)(react@19.2.4) + use-callback-ref: 1.3.3(@types/react@19.2.14)(react@19.2.4) + use-sidecar: 1.1.3(@types/react@19.2.14)(react@19.2.4) optionalDependencies: - '@types/react': 18.3.27 + '@types/react': 19.2.14 - react-grab@0.1.27(@types/react@18.3.27)(react@19.2.4): + react-grab@0.1.27(@types/react@19.2.14)(react@19.2.4): dependencies: '@medv/finder': 4.0.2 '@react-grab/cli': 0.1.27 - bippy: 0.5.32(@types/react@18.3.27)(react@19.2.4) + bippy: 0.5.32(@types/react@19.2.14)(react@19.2.4) solid-js: 1.9.11 optionalDependencies: react: 19.2.4 @@ -13378,12 +13306,12 @@ snapshots: react-refresh@0.18.0: {} - react-uid@2.3.0(@types/react@18.3.27)(react@19.2.4): + react-uid@2.3.0(@types/react@19.2.14)(react@19.2.4): dependencies: react: 19.2.4 tslib: 1.14.1 optionalDependencies: - '@types/react': 18.3.27 + '@types/react': 19.2.14 react-virtualized-auto-sizer@1.0.2(react-dom@19.2.4(react@19.2.4))(react@19.2.4): dependencies: @@ -13571,7 +13499,7 @@ snapshots: reusify@1.1.0: {} - rivetkit@2.1.6(@hono/node-server@1.19.9(hono@4.12.2))(@hono/node-ws@1.3.0(@hono/node-server@1.19.9(hono@4.12.2))(hono@4.12.2))(@standard-schema/spec@1.1.0)(drizzle-kit@0.31.9)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260313.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(better-sqlite3@11.10.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0))(ws@8.19.0): + rivetkit@2.1.6(@hono/node-server@1.19.9(hono@4.12.2))(@hono/node-ws@1.3.0(@hono/node-server@1.19.9(hono@4.12.2))(hono@4.12.2))(@standard-schema/spec@1.1.0)(drizzle-kit@0.31.9)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260317.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(better-sqlite3@11.10.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0))(ws@8.19.0): dependencies: '@hono/standard-validator': 0.1.5(@standard-schema/spec@1.1.0)(hono@4.12.2) '@hono/zod-openapi': 1.2.2(hono@4.12.2)(zod@4.3.6) @@ -13599,14 +13527,14 @@ snapshots: '@hono/node-server': 1.19.9(hono@4.12.2) '@hono/node-ws': 1.3.0(@hono/node-server@1.19.9(hono@4.12.2))(hono@4.12.2) drizzle-kit: 0.31.9 - drizzle-orm: 0.44.7(@cloudflare/workers-types@4.20260313.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(better-sqlite3@11.10.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0) + drizzle-orm: 0.44.7(@cloudflare/workers-types@4.20260317.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(better-sqlite3@11.10.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0) ws: 8.19.0 transitivePeerDependencies: - '@standard-schema/spec' - bufferutil - utf-8-validate - rivetkit@https://pkg.pr.new/rivet-dev/rivet/rivetkit@791500a(@e2b/code-interpreter@2.3.3)(@hono/node-server@1.19.9(hono@4.12.2))(@hono/node-ws@1.3.0(@hono/node-server@1.19.9(hono@4.12.2))(hono@4.12.2))(@standard-schema/spec@1.1.0)(dockerode@4.0.9)(drizzle-kit@0.31.9)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260313.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0))(ws@8.19.0): + rivetkit@https://pkg.pr.new/rivet-dev/rivet/rivetkit@791500a(@e2b/code-interpreter@2.3.3)(@hono/node-server@1.19.9(hono@4.12.2))(@hono/node-ws@1.3.0(@hono/node-server@1.19.9(hono@4.12.2))(hono@4.12.2))(@standard-schema/spec@1.1.0)(dockerode@4.0.9)(drizzle-kit@0.31.9)(drizzle-orm@0.44.7(@cloudflare/workers-types@4.20260317.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0))(ws@8.19.0): dependencies: '@hono/standard-validator': 0.1.5(@standard-schema/spec@1.1.0)(hono@4.12.2) '@hono/zod-openapi': 1.2.2(hono@4.12.2)(zod@4.3.6) @@ -13637,7 +13565,7 @@ snapshots: '@hono/node-ws': 1.3.0(@hono/node-server@1.19.9(hono@4.12.2))(hono@4.12.2) dockerode: 4.0.9 drizzle-kit: 0.31.9 - drizzle-orm: 0.44.7(@cloudflare/workers-types@4.20260313.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(better-sqlite3@11.10.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0) + drizzle-orm: 0.44.7(@cloudflare/workers-types@4.20260317.1)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.18.0)(better-sqlite3@11.10.0)(bun-types@1.3.10)(kysely@0.28.11)(pg@8.20.0) ws: 8.19.0 transitivePeerDependencies: - '@standard-schema/spec' @@ -14371,20 +14299,20 @@ snapshots: escalade: 3.2.0 picocolors: 1.1.1 - use-callback-ref@1.3.3(@types/react@18.3.27)(react@19.2.4): + use-callback-ref@1.3.3(@types/react@19.2.14)(react@19.2.4): dependencies: react: 19.2.4 tslib: 2.8.1 optionalDependencies: - '@types/react': 18.3.27 + '@types/react': 19.2.14 - use-sidecar@1.1.3(@types/react@18.3.27)(react@19.2.4): + use-sidecar@1.1.3(@types/react@19.2.14)(react@19.2.4): dependencies: detect-node-es: 1.1.0 react: 19.2.4 tslib: 2.8.1 optionalDependencies: - '@types/react': 18.3.27 + '@types/react': 19.2.14 use-sync-external-store@1.6.0(react@19.2.4): dependencies: @@ -14744,18 +14672,18 @@ snapshots: '@cloudflare/workerd-linux-arm64': 1.20260312.1 '@cloudflare/workerd-windows-64': 1.20260312.1 - wrangler@4.73.0(@cloudflare/workers-types@4.20260313.1): + wrangler@4.74.0(@cloudflare/workers-types@4.20260317.1): dependencies: '@cloudflare/kv-asset-handler': 0.4.2 '@cloudflare/unenv-preset': 2.15.0(unenv@2.0.0-rc.24)(workerd@1.20260312.1) blake3-wasm: 2.1.5 esbuild: 0.27.3 - miniflare: 4.20260312.0 + miniflare: 4.20260312.1 path-to-regexp: 6.3.0 unenv: 2.0.0-rc.24 workerd: 1.20260312.1 optionalDependencies: - '@cloudflare/workers-types': 4.20260313.1 + '@cloudflare/workers-types': 4.20260317.1 fsevents: 2.3.3 transitivePeerDependencies: - bufferutil diff --git a/research/desktop-streaming-architecture.md b/research/desktop-streaming-architecture.md new file mode 100644 index 0000000..f2c58fb --- /dev/null +++ b/research/desktop-streaming-architecture.md @@ -0,0 +1,103 @@ +# Desktop Streaming Architecture + +## Decision: neko over GStreamer (direct) and VNC + +We evaluated three approaches for streaming the virtual desktop to browser clients: + +1. **VNC (noVNC/websockify)** - traditional remote desktop +2. **GStreamer WebRTC (direct)** - custom GStreamer pipeline in the sandbox agent process +3. **neko** - standalone WebRTC streaming server with its own GStreamer pipeline + +We chose **neko**. + +## Approach comparison + +### VNC (noVNC) + +- Uses RFB protocol, not WebRTC. Relies on pixel-diff framebuffer updates over WebSocket. +- Higher latency than WebRTC (no hardware-accelerated codec, no adaptive bitrate). +- Requires a VNC server (x11vnc or similar) plus websockify for browser access. +- Input handling is mature but tied to the RFB protocol. +- No audio support without additional plumbing. + +**Rejected because:** Latency is noticeably worse than WebRTC-based approaches. The pixel-diff approach doesn't scale well at higher resolutions or frame rates. No native audio path. + +### GStreamer WebRTC (direct) + +- Custom pipeline: `ximagesrc -> videoconvert -> vp8enc -> rtpvp8pay -> webrtcbin`. +- Runs inside the sandbox agent Rust process using `gstreamer-rs` bindings. +- Requires feature-gating (`desktop-gstreamer` Cargo feature) and linking GStreamer at compile time. +- ICE candidate handling is complex: Docker-internal IPs (172.17.x.x) must be rewritten to 127.0.0.1 for host browser connectivity. +- UDP port range must be constrained via libnice NiceAgent properties to stay within Docker-forwarded ports. +- Input must be implemented separately (xdotool or custom X11 input injection). +- No built-in session management, authentication, or multi-client support. + +**Rejected because:** Too much complexity for the sandbox agent to own directly. ICE/NAT traversal bugs are hard to debug. The GStreamer Rust bindings add significant compile-time dependencies. Input handling requires a separate implementation. We built and tested this approach (branch `desktop-computer-use`, PR #226) and found: +- Black screen issues due to GStreamer pipeline negotiation failures +- ICE candidate rewriting fragility across Docker networking modes +- libnice port range configuration requires accessing internal NiceAgent properties that vary across GStreamer versions +- No data channel for low-latency input (had to fall back to WebSocket-based input which adds a round trip) + +### neko (chosen) + +- Standalone Go binary extracted from `ghcr.io/m1k1o/neko/base`. +- Has its own GStreamer pipeline internally (same `ximagesrc -> vp8enc -> webrtcbin` approach, but battle-tested). +- Provides WebSocket signaling, WebRTC media, and a binary data channel for input, all out of the box. +- Input via data channel is low-latency (sub-frame, no HTTP round trip). Uses X11 XTEST extension. +- Multi-session support with `noauth` provider (each browser tab gets its own session). +- ICE-lite mode with `--webrtc.nat1to1 127.0.0.1` eliminates NAT traversal issues for Docker-to-host. +- EPR (ephemeral port range) flag constrains UDP ports cleanly. +- Sandbox agent acts as a thin WebSocket proxy: browser WS connects to sandbox agent, which creates a per-connection neko login session and relays signaling messages bidirectionally. +- Audio codec support (opus) included for free. + +**Chosen because:** Neko encapsulates all the hard WebRTC/GStreamer/input complexity into a single binary. The sandbox agent only needs to: +1. Manage the neko process lifecycle (start/stop via the process runtime) +2. Proxy WebSocket signaling (bidirectional relay, ~60 lines of code) +3. Handle neko session creation (HTTP login to get a session cookie) + +This keeps the sandbox agent's desktop streaming code simple (~300 lines for the manager, ~120 lines for the WS proxy) while delivering production-quality WebRTC streaming with data channel input. + +## Architecture + +``` +Browser Sandbox Agent neko (internal) + | | | + |-- WS /stream/signaling --> |-- WS ws://127.0.0.1:18100/api/ws -->| + | | (bidirectional relay) | + |<-- neko signaling ---------|<-- neko signaling -------| + | | | + |<========= WebRTC (UDP 59000-59100) ==================>| + | VP8 video, Opus audio, binary data channel | + | | + |-- data channel input (mouse/keyboard) --------------->| + | (binary protocol: opcode + payload, big-endian) | +``` + +Key points: +- neko listens on internal port 18100 (not exposed externally). +- UDP ports 59000-59100 are forwarded through Docker for WebRTC media. +- `--webrtc.icelite` + `--webrtc.nat1to1 127.0.0.1` means neko advertises 127.0.0.1 as its ICE candidate, so the browser connects to localhost UDP ports directly. +- `--desktop.input.enabled=false` disables neko's custom xf86-input driver (not available outside neko's official Docker images). Input falls back to XTEST. +- Each WebSocket proxy connection creates a fresh neko login session with a unique username to avoid session conflicts when multiple clients connect. + +## Trade-offs + +| Concern | neko | GStreamer direct | +|---------|------|-----------------| +| Binary size | ~30MB additional binary | ~0 (uses system GStreamer libs) | +| Compile-time deps | None (external binary) | gstreamer-rs crate + GStreamer dev libs | +| Input latency | Sub-frame (data channel) | WebSocket round trip | +| ICE/NAT complexity | Handled by neko flags | Must implement in Rust | +| Multi-client | Built-in session management | Must implement | +| Maintenance | Upstream neko updates | Own all the code | +| Audio | Built-in (opus) | Must add audio pipeline | + +The main trade-off is the additional ~30MB binary size from neko. This is acceptable for the Docker-based deployment model where image size is less critical than reliability and development velocity. + +## References + +- neko v3: https://github.com/m1k1o/neko +- neko client reference: https://github.com/demodesk/neko-client +- neko data channel protocol: https://github.com/m1k1o/neko/blob/master/server/internal/webrtc/payload/receive.go +- GStreamer branch (closed): PR #226, branch `desktop-computer-use` +- Image digest: `ghcr.io/m1k1o/neko/base@sha256:0c384afa56268aaa2d5570211d284763d0840dcdd1a7d9a24be3081d94d3dfce` diff --git a/sdks/react/src/DesktopViewer.tsx b/sdks/react/src/DesktopViewer.tsx index f1ce711..5f55ed4 100644 --- a/sdks/react/src/DesktopViewer.tsx +++ b/sdks/react/src/DesktopViewer.tsx @@ -6,7 +6,7 @@ import type { DesktopMouseButton, DesktopStreamErrorStatus, DesktopStreamReadySt type ConnectionState = "connecting" | "ready" | "closed" | "error"; -export type DesktopViewerClient = Pick; +export type DesktopViewerClient = Pick; export interface DesktopViewerProps { client: DesktopViewerClient; @@ -51,7 +51,7 @@ const viewportStyle: CSSProperties = { background: "radial-gradient(circle at top, rgba(14, 165, 233, 0.18), transparent 45%), linear-gradient(180deg, #0f172a 0%, #111827 100%)", }; -const imageBaseStyle: CSSProperties = { +const videoBaseStyle: CSSProperties = { display: "block", width: "100%", height: "100%", @@ -78,112 +78,96 @@ const getStatusColor = (state: ConnectionState): string => { export const DesktopViewer = ({ client, className, style, imageStyle, height = 480, onConnect, onDisconnect, onError }: DesktopViewerProps) => { const wrapperRef = useRef(null); + const videoRef = useRef(null); const sessionRef = useRef | null>(null); const [connectionState, setConnectionState] = useState("connecting"); const [statusMessage, setStatusMessage] = useState("Starting desktop stream..."); - const [frameUrl, setFrameUrl] = useState(null); + const [hasVideo, setHasVideo] = useState(false); const [resolution, setResolution] = useState<{ width: number; height: number } | null>(null); useEffect(() => { let cancelled = false; - let lastObjectUrl: string | null = null; - let session: ReturnType | null = null; setConnectionState("connecting"); - setStatusMessage("Starting desktop stream..."); + setStatusMessage("Connecting to desktop stream..."); setResolution(null); + setHasVideo(false); - const connect = async () => { - try { - await client.startDesktopStream(); - if (cancelled) { - return; - } + const session = client.connectDesktopStream(); + sessionRef.current = session; - session = client.connectDesktopStream(); - sessionRef.current = session; - session.onReady((status) => { - if (cancelled) { - return; - } - setConnectionState("ready"); - setStatusMessage("Desktop stream connected."); - setResolution({ width: status.width, height: status.height }); - onConnect?.(status); - }); - session.onFrame((frame) => { - if (cancelled) { - return; - } - const nextUrl = URL.createObjectURL(new Blob([frame.slice().buffer], { type: "image/jpeg" })); - setFrameUrl((current) => { - if (current) { - URL.revokeObjectURL(current); - } - return nextUrl; - }); - if (lastObjectUrl) { - URL.revokeObjectURL(lastObjectUrl); - } - lastObjectUrl = nextUrl; - }); - session.onError((error) => { - if (cancelled) { - return; - } - setConnectionState("error"); - setStatusMessage(error instanceof Error ? error.message : error.message); - onError?.(error); - }); - session.onClose(() => { - if (cancelled) { - return; - } - setConnectionState((current) => (current === "error" ? current : "closed")); - setStatusMessage((current) => (current === "Desktop stream connected." ? "Desktop stream disconnected." : current)); - onDisconnect?.(); - }); - } catch (error) { - if (cancelled) { - return; - } - const nextError = error instanceof Error ? error : new Error("Failed to initialize desktop stream."); - setConnectionState("error"); - setStatusMessage(nextError.message); - onError?.(nextError); + session.onReady((status) => { + if (cancelled) return; + setConnectionState("ready"); + setStatusMessage("Desktop stream connected."); + setResolution({ width: status.width, height: status.height }); + onConnect?.(status); + }); + session.onTrack((stream) => { + if (cancelled) return; + const video = videoRef.current; + if (video) { + video.srcObject = stream; + void video.play().catch(() => undefined); + setHasVideo(true); } - }; - - void connect(); + }); + session.onError((error) => { + if (cancelled) return; + setConnectionState("error"); + setStatusMessage(error instanceof Error ? error.message : error.message); + onError?.(error); + }); + session.onDisconnect(() => { + if (cancelled) return; + setConnectionState((current) => (current === "error" ? current : "closed")); + setStatusMessage((current) => (current === "Desktop stream connected." ? "Desktop stream disconnected." : current)); + onDisconnect?.(); + }); return () => { cancelled = true; - session?.close(); + session.close(); sessionRef.current = null; - void client.stopDesktopStream().catch(() => undefined); - setFrameUrl((current) => { - if (current) { - URL.revokeObjectURL(current); - } - return null; - }); - if (lastObjectUrl) { - URL.revokeObjectURL(lastObjectUrl); + const video = videoRef.current; + if (video) { + video.srcObject = null; } + setHasVideo(false); }; }, [client, onConnect, onDisconnect, onError]); const scalePoint = (clientX: number, clientY: number) => { - const wrapper = wrapperRef.current; - if (!wrapper || !resolution) { + const video = videoRef.current; + if (!video || !resolution) { return null; } - const rect = wrapper.getBoundingClientRect(); + const rect = video.getBoundingClientRect(); if (rect.width === 0 || rect.height === 0) { return null; } - const x = Math.max(0, Math.min(resolution.width, ((clientX - rect.left) / rect.width) * resolution.width)); - const y = Math.max(0, Math.min(resolution.height, ((clientY - rect.top) / rect.height) * resolution.height)); + // The video uses objectFit: "contain", so we need to compute the actual + // rendered content area within the