(null);
// Active recording tracking
const activeRecording = useMemo(() => recordings.find((r) => r.status === "recording"), [recordings]);
+ const visibleWindows = useMemo(() => {
+ return windows.filter((win) => {
+ const title = win.title.trim();
+ if (win.isActive) return true;
+ if (!title || title === "Openbox") return false;
+ return win.width >= 120 && win.height >= 80;
+ });
+ }, [windows]);
const revokeScreenshotUrl = useCallback(() => {
setScreenshotUrl((current) => {
if (current?.startsWith("blob:") && typeof URL.revokeObjectURL === "function") {
@@ -103,6 +173,11 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => {
try {
const next = await getClient().getDesktopStatus();
setStatus(next);
+ // Status response now includes windows; sync them so we get window
+ // updates for free every time status is polled.
+ if (next.state === "active" && next.windows?.length) {
+ setWindows(next.windows);
+ }
return next;
} catch (loadError) {
setError(extractErrorMessage(loadError, "Unable to load desktop status."));
@@ -118,16 +193,36 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => {
setScreenshotLoading(true);
setScreenshotError(null);
try {
- const bytes = await getClient().takeDesktopScreenshot();
+ const quality = Number.parseInt(screenshotQuality, 10);
+ const scale = Number.parseFloat(screenshotScale);
+ const request: DesktopScreenshotRequest = {
+ format: screenshotFormat !== "png" ? screenshotFormat : undefined,
+ quality: screenshotFormat !== "png" && Number.isFinite(quality) ? quality : undefined,
+ scale: Number.isFinite(scale) && scale !== 1.0 ? scale : undefined,
+ showCursor: showCursor || undefined,
+ };
+ const bytes = await getClient().takeDesktopScreenshot(request);
revokeScreenshotUrl();
- setScreenshotUrl(await createScreenshotUrl(bytes));
+ const mimeType = screenshotFormat === "jpeg" ? "image/jpeg" : screenshotFormat === "webp" ? "image/webp" : "image/png";
+ setScreenshotUrl(await createScreenshotUrl(bytes, mimeType));
} catch (captureError) {
revokeScreenshotUrl();
setScreenshotError(extractErrorMessage(captureError, "Unable to capture desktop screenshot."));
} finally {
setScreenshotLoading(false);
}
- }, [getClient, revokeScreenshotUrl]);
+ }, [getClient, revokeScreenshotUrl, screenshotFormat, screenshotQuality, screenshotScale, showCursor]);
+ const loadMousePosition = useCallback(async () => {
+ setMousePosLoading(true);
+ try {
+ const pos = await getClient().getDesktopMousePosition();
+ setMousePos({ x: pos.x, y: pos.y });
+ } catch {
+ setMousePos(null);
+ } finally {
+ setMousePosLoading(false);
+ }
+ }, [getClient]);
const loadRecordings = useCallback(async () => {
setRecordingLoading(true);
setRecordingError(null);
@@ -140,15 +235,88 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => {
setRecordingLoading(false);
}
}, [getClient]);
+ const loadClipboard = useCallback(async () => {
+ setClipboardLoading(true);
+ setClipboardError(null);
+ try {
+ const result = await getClient().getDesktopClipboard({ selection: clipboardSelection });
+ setClipboardText(result.text);
+ } catch (err) {
+ setClipboardError(extractErrorMessage(err, "Unable to read clipboard."));
+ } finally {
+ setClipboardLoading(false);
+ }
+ }, [clipboardSelection, getClient]);
+ const loadWindows = useCallback(async () => {
+ setWindowsLoading(true);
+ setWindowsError(null);
+ try {
+ const result = await getClient().listDesktopWindows();
+ setWindows(result.windows);
+ } catch (err) {
+ setWindowsError(extractErrorMessage(err, "Unable to list windows."));
+ } finally {
+ setWindowsLoading(false);
+ }
+ }, [getClient]);
+ const handleFocusWindow = async (windowId: string) => {
+ setWindowActing(windowId);
+ try {
+ await getClient().focusDesktopWindow(windowId);
+ await loadWindows();
+ } catch (err) {
+ setWindowsError(extractErrorMessage(err, "Unable to focus window."));
+ } finally {
+ setWindowActing(null);
+ }
+ };
+ const handleMoveWindow = async (windowId: string) => {
+ const x = Number.parseInt(editX, 10);
+ const y = Number.parseInt(editY, 10);
+ if (!Number.isFinite(x) || !Number.isFinite(y)) return;
+ setWindowActing(windowId);
+ try {
+ await getClient().moveDesktopWindow(windowId, { x, y });
+ setEditingWindow(null);
+ await loadWindows();
+ } catch (err) {
+ setWindowsError(extractErrorMessage(err, "Unable to move window."));
+ } finally {
+ setWindowActing(null);
+ }
+ };
+ const handleResizeWindow = async (windowId: string) => {
+ const nextWidth = Number.parseInt(editW, 10);
+ const nextHeight = Number.parseInt(editH, 10);
+ if (!Number.isFinite(nextWidth) || !Number.isFinite(nextHeight) || nextWidth <= 0 || nextHeight <= 0) return;
+ setWindowActing(windowId);
+ try {
+ await getClient().resizeDesktopWindow(windowId, { width: nextWidth, height: nextHeight });
+ setEditingWindow(null);
+ await loadWindows();
+ } catch (err) {
+ setWindowsError(extractErrorMessage(err, "Unable to resize window."));
+ } finally {
+ setWindowActing(null);
+ }
+ };
useEffect(() => {
void loadStatus();
}, [loadStatus]);
+ // Auto-refresh status (and windows via status) every 5 seconds when active
+ useEffect(() => {
+ if (status?.state !== "active") return;
+ const interval = setInterval(() => void loadStatus("refresh"), 5000);
+ return () => clearInterval(interval);
+ }, [status?.state, loadStatus]);
useEffect(() => {
if (status?.state === "active") {
void loadRecordings();
} else {
revokeScreenshotUrl();
setLiveViewActive(false);
+ setMousePos(null);
+ setEditingWindow(null);
}
}, [status?.state, loadRecordings, revokeScreenshotUrl]);
useEffect(() => {
@@ -160,19 +328,35 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => {
const interval = setInterval(() => void loadRecordings(), 3000);
return () => clearInterval(interval);
}, [activeRecording, loadRecordings]);
+ useEffect(() => {
+ if (status?.state !== "active") {
+ setWindows([]);
+ return;
+ }
+ // Initial load; subsequent updates come from the status auto-refresh.
+ void loadWindows();
+ }, [status?.state, loadWindows]);
const handleStart = async () => {
const parsedWidth = Number.parseInt(width, 10);
const parsedHeight = Number.parseInt(height, 10);
const parsedDpi = Number.parseInt(dpi, 10);
+ const parsedFrameRate = Number.parseInt(streamFrameRate, 10);
+ const parsedRecordingFps = Number.parseInt(defaultRecordingFps, 10);
setActing("start");
setError(null);
const startedAt = Date.now();
try {
- const next = await getClient().startDesktop({
+ const request: DesktopStartRequestWithAdvanced = {
width: Number.isFinite(parsedWidth) ? parsedWidth : undefined,
height: Number.isFinite(parsedHeight) ? parsedHeight : undefined,
dpi: Number.isFinite(parsedDpi) ? parsedDpi : undefined,
- });
+ streamVideoCodec: streamVideoCodec !== "vp8" ? streamVideoCodec : undefined,
+ streamAudioCodec: streamAudioCodec !== "opus" ? streamAudioCodec : undefined,
+ streamFrameRate: Number.isFinite(parsedFrameRate) && parsedFrameRate !== 30 ? parsedFrameRate : undefined,
+ webrtcPortRange: webrtcPortRange !== "59050-59070" ? webrtcPortRange : undefined,
+ recordingFps: Number.isFinite(parsedRecordingFps) && parsedRecordingFps !== 30 ? parsedRecordingFps : undefined,
+ };
+ const next = await getClient().startDesktop(request);
setStatus(next);
} catch (startError) {
setError(extractErrorMessage(startError, "Unable to start desktop runtime."));
@@ -205,6 +389,18 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => {
setActing(null);
}
};
+ const handleWriteClipboard = async () => {
+ setClipboardWriting(true);
+ setClipboardError(null);
+ try {
+ await getClient().setDesktopClipboard({ text: clipboardWriteText, selection: clipboardSelection });
+ setClipboardText(clipboardWriteText);
+ } catch (err) {
+ setClipboardError(extractErrorMessage(err, "Unable to write clipboard."));
+ } finally {
+ setClipboardWriting(false);
+ }
+ };
const handleStartRecording = async () => {
const fps = Number.parseInt(recordingFps, 10);
setRecordingActing("start");
@@ -262,6 +458,41 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => {
setDownloadingRecordingId(null);
}
};
+ const handleLaunchApp = async () => {
+ if (!launchApp.trim()) return;
+ setLaunching(true);
+ setLaunchError(null);
+ setLaunchResult(null);
+ try {
+ const args = launchArgs.trim() ? launchArgs.trim().split(/\s+/) : undefined;
+ const result = await getClient().launchDesktopApp({
+ app: launchApp.trim(),
+ args,
+ wait: launchWait || undefined,
+ });
+ setLaunchResult(`Started ${result.processId}${result.windowId ? ` (window: ${result.windowId})` : ""}`);
+ await loadWindows();
+ } catch (err) {
+ setLaunchError(extractErrorMessage(err, "Unable to launch app."));
+ } finally {
+ setLaunching(false);
+ }
+ };
+ const handleOpenTarget = async () => {
+ if (!openTarget.trim()) return;
+ setOpening(true);
+ setOpenError(null);
+ setOpenResult(null);
+ try {
+ const result = await getClient().openDesktopTarget({ target: openTarget.trim() });
+ setOpenResult(`Opened via ${result.processId}`);
+ await loadWindows();
+ } catch (err) {
+ setOpenError(extractErrorMessage(err, "Unable to open target."));
+ } finally {
+ setOpening(false);
+ }
+ };
const canRefreshScreenshot = status?.state === "active";
const isActive = status?.state === "active";
const resolutionLabel = useMemo(() => {
@@ -284,6 +515,48 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => {
)}
+ {isActive && !liveViewActive && (
+
+ )}
{error && {error}
}
{screenshotError && {screenshotError}
}
{/* ========== Runtime Section ========== */}
@@ -329,6 +602,56 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => {
setDpi(event.target.value)} inputMode="numeric" />
+
+ {showAdvancedStart && (
+
+
+
+
+
+
+
+
+
+
+
+ setStreamFrameRate(event.target.value)}
+ inputMode="numeric"
+ disabled={isActive}
+ />
+
+
+
+ setWebrtcPortRange(event.target.value)} disabled={isActive} />
+
+
+
+ setDefaultRecordingFps(event.target.value)}
+ inputMode="numeric"
+ disabled={isActive}
+ />
+
+
{isActive ? (
)}
{!isActive && Start the desktop runtime to enable live view.
}
- {isActive && liveViewActive && }
+ {isActive && liveViewActive && (
+ <>
+
+ Right click to open window
+ {status?.resolution && (
+
+ {status.resolution.width}x{status.resolution.height}
+
+ )}
+
+
+
+ )}
{isActive && !liveViewActive && (
<>
{screenshotUrl ? (
@@ -428,7 +773,313 @@ const DesktopTab = ({ getClient }: { getClient: () => SandboxAgent }) => {
)}
)}
+ {isActive && (
+
+
+ {mousePos && (
+
+ ({mousePos.x}, {mousePos.y})
+
+ )}
+
+ )}
+ {isActive && (
+
+
+
+
+ Clipboard
+
+
+
+
+
+
+ {clipboardError && (
+
+ {clipboardError}
+
+ )}
+
+
Current contents
+
+ {clipboardText ? clipboardText : (empty)}
+
+
+