import { AppWindow, Camera, Circle, Clipboard, Download, ExternalLink, Loader2, Monitor, MousePointer, Play, RefreshCw, Square, Trash2, Video, } from "lucide-react"; import { useCallback, useEffect, useMemo, useState } from "react"; import { SandboxAgentError } from "sandbox-agent"; import type { DesktopRecordingInfo, DesktopStatusResponse, DesktopWindowInfo, SandboxAgent } from "sandbox-agent"; import { DesktopViewer } from "@sandbox-agent/react"; import type { DesktopViewerClient } from "@sandbox-agent/react"; const MIN_SPIN_MS = 350; type DesktopScreenshotRequest = Parameters[0] & { showCursor?: boolean; }; type DesktopStartRequestWithAdvanced = Parameters[0] & { streamVideoCodec?: string; streamAudioCodec?: string; streamFrameRate?: number; webrtcPortRange?: string; recordingFps?: number; }; const extractErrorMessage = (error: unknown, fallback: string): string => { if (error instanceof SandboxAgentError && error.problem?.detail) return error.problem.detail; if (error instanceof Error) return error.message; return fallback; }; const formatStartedAt = (value: string | null | undefined): string => { if (!value) return "Not started"; const parsed = new Date(value); return Number.isNaN(parsed.getTime()) ? value : parsed.toLocaleString(); }; const formatBytes = (bytes: number): string => { if (bytes === 0) return "0 B"; const units = ["B", "KB", "MB", "GB"]; const i = Math.floor(Math.log(bytes) / Math.log(1024)); return `${(bytes / 1024 ** i).toFixed(i > 0 ? 1 : 0)} ${units[i]}`; }; const formatDuration = (start: string, end?: string | null): string => { const startMs = new Date(start).getTime(); const endMs = end ? new Date(end).getTime() : Date.now(); if (Number.isNaN(startMs) || Number.isNaN(endMs)) return "Unknown"; const seconds = Math.round((endMs - startMs) / 1000); if (seconds < 60) return `${seconds}s`; const mins = Math.floor(seconds / 60); const secs = seconds % 60; return `${mins}m ${secs}s`; }; const createScreenshotUrl = async (bytes: Uint8Array, mimeType = "image/png"): Promise => { const payload = new Uint8Array(bytes.byteLength); payload.set(bytes); const blob = new Blob([payload.buffer], { type: mimeType }); if (typeof URL.createObjectURL === "function") { return URL.createObjectURL(blob); } return await new Promise((resolve, reject) => { const reader = new FileReader(); reader.onerror = () => reject(reader.error ?? new Error("Unable to read screenshot blob.")); reader.onload = () => { if (typeof reader.result === "string") { resolve(reader.result); } else { reject(new Error("Unable to read screenshot blob.")); } }; reader.readAsDataURL(blob); }); }; const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => { const [status, setStatus] = useState(null); const [loading, setLoading] = useState(false); const [refreshing, setRefreshing] = useState(false); const [acting, setActing] = useState<"start" | "stop" | null>(null); const [error, setError] = useState(null); const [width, setWidth] = useState("1440"); const [height, setHeight] = useState("900"); const [dpi, setDpi] = useState("96"); // Screenshot fallback const [screenshotUrl, setScreenshotUrl] = useState(null); const [screenshotLoading, setScreenshotLoading] = useState(false); const [screenshotError, setScreenshotError] = useState(null); const [screenshotFormat, setScreenshotFormat] = useState<"png" | "jpeg" | "webp">("png"); const [screenshotQuality, setScreenshotQuality] = useState("85"); const [screenshotScale, setScreenshotScale] = useState("1.0"); const [showCursor, setShowCursor] = useState(false); // Live view const [liveViewActive, setLiveViewActive] = useState(false); const [liveViewError, setLiveViewError] = useState(null); const [mousePos, setMousePos] = useState<{ x: number; y: number } | null>(null); const [mousePosLoading, setMousePosLoading] = useState(false); // Memoize the client as a DesktopViewerClient so the reference is stable // across renders and doesn't cause the DesktopViewer effect to re-fire. const viewerClient = useMemo(() => { const c = getClient(); return { startDesktopStream: () => c.startDesktopStream(), stopDesktopStream: () => c.stopDesktopStream(), connectDesktopStream: (opts?: Parameters[0]) => c.connectDesktopStream(opts), }; }, [getClient]); // Recording const [recordings, setRecordings] = useState([]); const [recordingLoading, setRecordingLoading] = useState(false); const [recordingActing, setRecordingActing] = useState<"start" | "stop" | null>(null); const [recordingError, setRecordingError] = useState(null); const [recordingFps, setRecordingFps] = useState("30"); const [deletingRecordingId, setDeletingRecordingId] = useState(null); const [downloadingRecordingId, setDownloadingRecordingId] = useState(null); const [showAdvancedStart, setShowAdvancedStart] = useState(false); const [streamVideoCodec, setStreamVideoCodec] = useState("vp8"); const [streamAudioCodec, setStreamAudioCodec] = useState("opus"); const [streamFrameRate, setStreamFrameRate] = useState("30"); const [webrtcPortRange, setWebrtcPortRange] = useState("59050-59070"); const [defaultRecordingFps, setDefaultRecordingFps] = useState("30"); const [clipboardText, setClipboardText] = useState(""); const [clipboardSelection, setClipboardSelection] = useState<"clipboard" | "primary">("clipboard"); const [clipboardLoading, setClipboardLoading] = useState(false); const [clipboardError, setClipboardError] = useState(null); const [clipboardWriteText, setClipboardWriteText] = useState(""); const [clipboardWriting, setClipboardWriting] = useState(false); const [windows, setWindows] = useState([]); const [windowsLoading, setWindowsLoading] = useState(false); const [windowsError, setWindowsError] = useState(null); const [windowActing, setWindowActing] = useState(null); const [editingWindow, setEditingWindow] = useState<{ id: string; action: "move" | "resize" } | null>(null); const [editX, setEditX] = useState(""); const [editY, setEditY] = useState(""); const [editW, setEditW] = useState(""); const [editH, setEditH] = useState(""); const [launchApp, setLaunchApp] = useState("firefox"); const [launchArgs, setLaunchArgs] = useState(""); const [launchWait, setLaunchWait] = useState(true); const [launching, setLaunching] = useState(false); const [launchResult, setLaunchResult] = useState(null); const [launchError, setLaunchError] = useState(null); const [openTarget, setOpenTarget] = useState(""); const [opening, setOpening] = useState(false); const [openResult, setOpenResult] = useState(null); const [openError, setOpenError] = useState(null); // Active recording tracking const activeRecording = useMemo(() => recordings.find((r) => r.status === "recording"), [recordings]); const visibleWindows = useMemo(() => { return windows.filter((win) => { const title = win.title.trim(); if (win.isActive) return true; if (!title || title === "Openbox") return false; return win.width >= 120 && win.height >= 80; }); }, [windows]); const revokeScreenshotUrl = useCallback(() => { setScreenshotUrl((current) => { if (current?.startsWith("blob:") && typeof URL.revokeObjectURL === "function") { URL.revokeObjectURL(current); } return null; }); }, []); const loadStatus = useCallback( async (mode: "initial" | "refresh" = "initial") => { if (mode === "initial") setLoading(true); else setRefreshing(true); setError(null); try { const next = await getClient().getDesktopStatus(); setStatus(next); // Status response now includes windows; sync them so we get window // updates for free every time status is polled. if (next.state === "active" && next.windows?.length) { setWindows(next.windows); } return next; } catch (loadError) { setError(extractErrorMessage(loadError, "Unable to load desktop status.")); return null; } finally { setLoading(false); setRefreshing(false); } }, [getClient], ); const refreshScreenshot = useCallback(async () => { setScreenshotLoading(true); setScreenshotError(null); try { const quality = Number.parseInt(screenshotQuality, 10); const scale = Number.parseFloat(screenshotScale); const request: DesktopScreenshotRequest = { format: screenshotFormat !== "png" ? screenshotFormat : undefined, quality: screenshotFormat !== "png" && Number.isFinite(quality) ? quality : undefined, scale: Number.isFinite(scale) && scale !== 1.0 ? scale : undefined, showCursor: showCursor || undefined, }; const bytes = await getClient().takeDesktopScreenshot(request); revokeScreenshotUrl(); const mimeType = screenshotFormat === "jpeg" ? "image/jpeg" : screenshotFormat === "webp" ? "image/webp" : "image/png"; setScreenshotUrl(await createScreenshotUrl(bytes, mimeType)); } catch (captureError) { revokeScreenshotUrl(); setScreenshotError(extractErrorMessage(captureError, "Unable to capture desktop screenshot.")); } finally { setScreenshotLoading(false); } }, [getClient, revokeScreenshotUrl, screenshotFormat, screenshotQuality, screenshotScale, showCursor]); const loadMousePosition = useCallback(async () => { setMousePosLoading(true); try { const pos = await getClient().getDesktopMousePosition(); setMousePos({ x: pos.x, y: pos.y }); } catch { setMousePos(null); } finally { setMousePosLoading(false); } }, [getClient]); const loadRecordings = useCallback(async () => { setRecordingLoading(true); setRecordingError(null); try { const result = await getClient().listDesktopRecordings(); setRecordings(result.recordings); } catch (loadError) { setRecordingError(extractErrorMessage(loadError, "Unable to load recordings.")); } finally { setRecordingLoading(false); } }, [getClient]); const loadClipboard = useCallback(async () => { setClipboardLoading(true); setClipboardError(null); try { const result = await getClient().getDesktopClipboard({ selection: clipboardSelection }); setClipboardText(result.text); } catch (err) { setClipboardError(extractErrorMessage(err, "Unable to read clipboard.")); } finally { setClipboardLoading(false); } }, [clipboardSelection, getClient]); const loadWindows = useCallback(async () => { setWindowsLoading(true); setWindowsError(null); try { const result = await getClient().listDesktopWindows(); setWindows(result.windows); } catch (err) { setWindowsError(extractErrorMessage(err, "Unable to list windows.")); } finally { setWindowsLoading(false); } }, [getClient]); const handleFocusWindow = async (windowId: string) => { setWindowActing(windowId); try { await getClient().focusDesktopWindow(windowId); await loadWindows(); } catch (err) { setWindowsError(extractErrorMessage(err, "Unable to focus window.")); } finally { setWindowActing(null); } }; const handleMoveWindow = async (windowId: string) => { const x = Number.parseInt(editX, 10); const y = Number.parseInt(editY, 10); if (!Number.isFinite(x) || !Number.isFinite(y)) return; setWindowActing(windowId); try { await getClient().moveDesktopWindow(windowId, { x, y }); setEditingWindow(null); await loadWindows(); } catch (err) { setWindowsError(extractErrorMessage(err, "Unable to move window.")); } finally { setWindowActing(null); } }; const handleResizeWindow = async (windowId: string) => { const nextWidth = Number.parseInt(editW, 10); const nextHeight = Number.parseInt(editH, 10); if (!Number.isFinite(nextWidth) || !Number.isFinite(nextHeight) || nextWidth <= 0 || nextHeight <= 0) return; setWindowActing(windowId); try { await getClient().resizeDesktopWindow(windowId, { width: nextWidth, height: nextHeight }); setEditingWindow(null); await loadWindows(); } catch (err) { setWindowsError(extractErrorMessage(err, "Unable to resize window.")); } finally { setWindowActing(null); } }; useEffect(() => { void loadStatus(); }, [loadStatus]); // Auto-refresh status (and windows via status) every 5 seconds when active useEffect(() => { if (status?.state !== "active") return; const interval = setInterval(() => void loadStatus("refresh"), 5000); return () => clearInterval(interval); }, [status?.state, loadStatus]); useEffect(() => { if (status?.state === "active") { void loadRecordings(); } else { revokeScreenshotUrl(); setLiveViewActive(false); setMousePos(null); setEditingWindow(null); } }, [status?.state, loadRecordings, revokeScreenshotUrl]); useEffect(() => { return () => revokeScreenshotUrl(); }, [revokeScreenshotUrl]); // Poll recording list while a recording is active useEffect(() => { if (!activeRecording) return; const interval = setInterval(() => void loadRecordings(), 3000); return () => clearInterval(interval); }, [activeRecording, loadRecordings]); useEffect(() => { if (status?.state !== "active") { setWindows([]); return; } // Initial load; subsequent updates come from the status auto-refresh. void loadWindows(); }, [status?.state, loadWindows]); const handleStart = async () => { const parsedWidth = Number.parseInt(width, 10); const parsedHeight = Number.parseInt(height, 10); const parsedDpi = Number.parseInt(dpi, 10); const parsedFrameRate = Number.parseInt(streamFrameRate, 10); const parsedRecordingFps = Number.parseInt(defaultRecordingFps, 10); setActing("start"); setError(null); const startedAt = Date.now(); try { const request: DesktopStartRequestWithAdvanced = { width: Number.isFinite(parsedWidth) ? parsedWidth : undefined, height: Number.isFinite(parsedHeight) ? parsedHeight : undefined, dpi: Number.isFinite(parsedDpi) ? parsedDpi : undefined, streamVideoCodec: streamVideoCodec !== "vp8" ? streamVideoCodec : undefined, streamAudioCodec: streamAudioCodec !== "opus" ? streamAudioCodec : undefined, streamFrameRate: Number.isFinite(parsedFrameRate) && parsedFrameRate !== 30 ? parsedFrameRate : undefined, webrtcPortRange: webrtcPortRange !== "59050-59070" ? webrtcPortRange : undefined, recordingFps: Number.isFinite(parsedRecordingFps) && parsedRecordingFps !== 30 ? parsedRecordingFps : undefined, }; const next = await getClient().startDesktop(request); setStatus(next); } catch (startError) { setError(extractErrorMessage(startError, "Unable to start desktop runtime.")); await loadStatus("refresh"); } finally { const elapsedMs = Date.now() - startedAt; if (elapsedMs < MIN_SPIN_MS) { await new Promise((resolve) => window.setTimeout(resolve, MIN_SPIN_MS - elapsedMs)); } setActing(null); } }; const handleStop = async () => { setActing("stop"); setError(null); const startedAt = Date.now(); try { const next = await getClient().stopDesktop(); setStatus(next); revokeScreenshotUrl(); setLiveViewActive(false); } catch (stopError) { setError(extractErrorMessage(stopError, "Unable to stop desktop runtime.")); await loadStatus("refresh"); } finally { const elapsedMs = Date.now() - startedAt; if (elapsedMs < MIN_SPIN_MS) { await new Promise((resolve) => window.setTimeout(resolve, MIN_SPIN_MS - elapsedMs)); } setActing(null); } }; const handleWriteClipboard = async () => { setClipboardWriting(true); setClipboardError(null); try { await getClient().setDesktopClipboard({ text: clipboardWriteText, selection: clipboardSelection }); setClipboardText(clipboardWriteText); } catch (err) { setClipboardError(extractErrorMessage(err, "Unable to write clipboard.")); } finally { setClipboardWriting(false); } }; const handleStartRecording = async () => { const fps = Number.parseInt(recordingFps, 10); setRecordingActing("start"); setRecordingError(null); try { await getClient().startDesktopRecording({ fps: Number.isFinite(fps) && fps > 0 ? fps : undefined, }); await loadRecordings(); } catch (err) { setRecordingError(extractErrorMessage(err, "Unable to start recording.")); } finally { setRecordingActing(null); } }; const handleStopRecording = async () => { setRecordingActing("stop"); setRecordingError(null); try { await getClient().stopDesktopRecording(); await loadRecordings(); } catch (err) { setRecordingError(extractErrorMessage(err, "Unable to stop recording.")); } finally { setRecordingActing(null); } }; const handleDeleteRecording = async (id: string) => { setDeletingRecordingId(id); try { await getClient().deleteDesktopRecording(id); setRecordings((prev) => prev.filter((r) => r.id !== id)); } catch (err) { setRecordingError(extractErrorMessage(err, "Unable to delete recording.")); } finally { setDeletingRecordingId(null); } }; const handleDownloadRecording = async (id: string, fileName: string) => { setDownloadingRecordingId(id); try { const bytes = await getClient().downloadDesktopRecording(id); const blob = new Blob([bytes], { type: "video/mp4" }); const url = URL.createObjectURL(blob); const a = document.createElement("a"); a.href = url; a.download = fileName; document.body.appendChild(a); a.click(); document.body.removeChild(a); URL.revokeObjectURL(url); } catch (err) { setRecordingError(extractErrorMessage(err, "Unable to download recording.")); } finally { setDownloadingRecordingId(null); } }; const handleLaunchApp = async () => { if (!launchApp.trim()) return; setLaunching(true); setLaunchError(null); setLaunchResult(null); try { const args = launchArgs.trim() ? launchArgs.trim().split(/\s+/) : undefined; const result = await getClient().launchDesktopApp({ app: launchApp.trim(), args, wait: launchWait || undefined, }); setLaunchResult(`Started ${result.processId}${result.windowId ? ` (window: ${result.windowId})` : ""}`); await loadWindows(); } catch (err) { setLaunchError(extractErrorMessage(err, "Unable to launch app.")); } finally { setLaunching(false); } }; const handleOpenTarget = async () => { if (!openTarget.trim()) return; setOpening(true); setOpenError(null); setOpenResult(null); try { const result = await getClient().openDesktopTarget({ target: openTarget.trim() }); setOpenResult(`Opened via ${result.processId}`); await loadWindows(); } catch (err) { setOpenError(extractErrorMessage(err, "Unable to open target.")); } finally { setOpening(false); } }; const canRefreshScreenshot = status?.state === "active"; const isActive = status?.state === "active"; const resolutionLabel = useMemo(() => { const resolution = status?.resolution; if (!resolution) return "Unknown"; const dpiLabel = resolution.dpi ? ` @ ${resolution.dpi} DPI` : ""; return `${resolution.width} x ${resolution.height}${dpiLabel}`; }, [status?.resolution]); return (
{isActive && !liveViewActive && ( )}
{isActive && !liveViewActive && (
{screenshotFormat !== "png" && (
setScreenshotQuality(event.target.value)} inputMode="numeric" style={{ maxWidth: 60 }} />
)}
setScreenshotScale(event.target.value)} inputMode="decimal" style={{ maxWidth: 60 }} />
)} {error &&
{error}
} {screenshotError &&
{screenshotError}
} {/* ========== Runtime Section ========== */}
Desktop Runtime {status?.state ?? "unknown"}
Display
{status?.display ?? "Not assigned"}
Resolution
{resolutionLabel}
Started
{formatStartedAt(status?.startedAt)}
setWidth(event.target.value)} inputMode="numeric" />
setHeight(event.target.value)} inputMode="numeric" />
setDpi(event.target.value)} inputMode="numeric" />
{showAdvancedStart && (
setStreamFrameRate(event.target.value)} inputMode="numeric" disabled={isActive} />
setWebrtcPortRange(event.target.value)} disabled={isActive} />
setDefaultRecordingFps(event.target.value)} inputMode="numeric" disabled={isActive} />
)}
{isActive ? ( ) : ( )}
{/* ========== Missing Dependencies ========== */} {status?.missingDependencies && status.missingDependencies.length > 0 && (
Missing Dependencies
{status.missingDependencies.map((dependency) => ( {dependency} ))}
{status.installCommand && ( <>
Install command
{status.installCommand}
)}
)} {/* ========== Live View Section ========== */}
{isActive && ( )}
{liveViewError && (
{liveViewError}
)} {!isActive &&
Start the desktop runtime to enable live view.
} {isActive && liveViewActive && ( <>
Right click to open window {status?.resolution && ( {status.resolution.width}x{status.resolution.height} )}
)} {isActive && !liveViewActive && ( <> {screenshotUrl ? (
Desktop screenshot
) : (
Click "Start Stream" for live desktop view, or use the Screenshot button above.
)} )} {isActive && (
{mousePos && ( ({mousePos.x}, {mousePos.y}) )}
)}
{isActive && (
Clipboard
{clipboardError && (
{clipboardError}
)}
Current contents
              {clipboardText ? clipboardText : (empty)}
            
Write to clipboard