mirror of
https://github.com/harivansh-afk/sandbox-agent.git
synced 2026-04-15 06:04:43 +00:00
feat: [US-027] - Add Browser tab - screenshot, tabs, and console sections
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Nathan Flurry
parent
4c00d71f5d
commit
589362ffc0
1 changed files with 431 additions and 3 deletions
|
|
@ -1,7 +1,7 @@
|
|||
import { ArrowLeft, ArrowRight, Globe, Loader2, Play, RefreshCw, Square } from "lucide-react";
|
||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
import { ArrowLeft, ArrowRight, Camera, Globe, Layers, Loader2, Play, Plus, RefreshCw, Square, Terminal, X } from "lucide-react";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { SandboxAgentError } from "sandbox-agent";
|
||||
import type { BrowserContextInfo, BrowserStatusResponse, SandboxAgent } from "sandbox-agent";
|
||||
import type { BrowserConsoleMessage, BrowserContextInfo, BrowserStatusResponse, BrowserTabInfo, SandboxAgent } from "sandbox-agent";
|
||||
import { DesktopViewer } from "@sandbox-agent/react";
|
||||
import type { BrowserViewerClient } from "@sandbox-agent/react";
|
||||
|
||||
|
|
@ -19,6 +19,42 @@ const formatStartedAt = (value: string | null | undefined): string => {
|
|||
return Number.isNaN(parsed.getTime()) ? value : parsed.toLocaleString();
|
||||
};
|
||||
|
||||
const createScreenshotUrl = async (bytes: Uint8Array, mimeType = "image/png"): Promise<string> => {
|
||||
const payload = new Uint8Array(bytes.byteLength);
|
||||
payload.set(bytes);
|
||||
const blob = new Blob([payload.buffer], { type: mimeType });
|
||||
if (typeof URL.createObjectURL === "function") {
|
||||
return URL.createObjectURL(blob);
|
||||
}
|
||||
return await new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onerror = () => reject(reader.error ?? new Error("Unable to read screenshot blob."));
|
||||
reader.onload = () => {
|
||||
if (typeof reader.result === "string") {
|
||||
resolve(reader.result);
|
||||
} else {
|
||||
reject(new Error("Unable to read screenshot blob."));
|
||||
}
|
||||
};
|
||||
reader.readAsDataURL(blob);
|
||||
});
|
||||
};
|
||||
|
||||
const CONSOLE_LEVELS = ["all", "log", "warn", "error", "info"] as const;
|
||||
|
||||
const consoleLevelColor = (level: string): string => {
|
||||
switch (level) {
|
||||
case "error":
|
||||
return "var(--danger, #ef4444)";
|
||||
case "warning":
|
||||
return "var(--warning, #f59e0b)";
|
||||
case "info":
|
||||
return "var(--info, #3b82f6)";
|
||||
default:
|
||||
return "var(--muted)";
|
||||
}
|
||||
};
|
||||
|
||||
const BrowserTab = ({ getClient }: { getClient: () => SandboxAgent }) => {
|
||||
// Status
|
||||
const [status, setStatus] = useState<BrowserStatusResponse | null>(null);
|
||||
|
|
@ -34,6 +70,29 @@ const BrowserTab = ({ getClient }: { getClient: () => SandboxAgent }) => {
|
|||
const [contextId, setContextId] = useState("");
|
||||
const [contexts, setContexts] = useState<BrowserContextInfo[]>([]);
|
||||
|
||||
// Screenshot
|
||||
const [screenshotUrl, setScreenshotUrl] = useState<string | null>(null);
|
||||
const [screenshotLoading, setScreenshotLoading] = useState(false);
|
||||
const [screenshotError, setScreenshotError] = useState<string | null>(null);
|
||||
const [screenshotFormat, setScreenshotFormat] = useState<"png" | "jpeg" | "webp">("png");
|
||||
const [screenshotQuality, setScreenshotQuality] = useState("85");
|
||||
const [screenshotFullPage, setScreenshotFullPage] = useState(false);
|
||||
const [screenshotSelector, setScreenshotSelector] = useState("");
|
||||
|
||||
// Tabs
|
||||
const [tabs, setTabs] = useState<BrowserTabInfo[]>([]);
|
||||
const [tabsLoading, setTabsLoading] = useState(false);
|
||||
const [tabsError, setTabsError] = useState<string | null>(null);
|
||||
const [newTabUrl, setNewTabUrl] = useState("");
|
||||
const [tabActing, setTabActing] = useState<string | null>(null);
|
||||
|
||||
// Console
|
||||
const [consoleMessages, setConsoleMessages] = useState<BrowserConsoleMessage[]>([]);
|
||||
const [consoleLoading, setConsoleLoading] = useState(false);
|
||||
const [consoleError, setConsoleError] = useState<string | null>(null);
|
||||
const [consoleLevel, setConsoleLevel] = useState<string>("all");
|
||||
const consoleEndRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
// Live view
|
||||
const [liveViewActive, setLiveViewActive] = useState(false);
|
||||
const [liveViewError, setLiveViewError] = useState<string | null>(null);
|
||||
|
|
@ -90,6 +149,108 @@ const BrowserTab = ({ getClient }: { getClient: () => SandboxAgent }) => {
|
|||
}
|
||||
}, [getClient]);
|
||||
|
||||
// Screenshot
|
||||
const revokeScreenshotUrl = useCallback(() => {
|
||||
setScreenshotUrl((current) => {
|
||||
if (current?.startsWith("blob:") && typeof URL.revokeObjectURL === "function") {
|
||||
URL.revokeObjectURL(current);
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}, []);
|
||||
|
||||
const refreshScreenshot = useCallback(async () => {
|
||||
setScreenshotLoading(true);
|
||||
setScreenshotError(null);
|
||||
try {
|
||||
const quality = Number.parseInt(screenshotQuality, 10);
|
||||
const request: Parameters<SandboxAgent["takeBrowserScreenshot"]>[0] = {
|
||||
format: screenshotFormat !== "png" ? screenshotFormat : undefined,
|
||||
quality: screenshotFormat !== "png" && Number.isFinite(quality) ? quality : undefined,
|
||||
fullPage: screenshotFullPage || undefined,
|
||||
selector: screenshotSelector.trim() || undefined,
|
||||
};
|
||||
const bytes = await getClient().takeBrowserScreenshot(request);
|
||||
revokeScreenshotUrl();
|
||||
const mimeType = screenshotFormat === "jpeg" ? "image/jpeg" : screenshotFormat === "webp" ? "image/webp" : "image/png";
|
||||
setScreenshotUrl(await createScreenshotUrl(bytes, mimeType));
|
||||
} catch (captureError) {
|
||||
revokeScreenshotUrl();
|
||||
setScreenshotError(extractErrorMessage(captureError, "Unable to capture browser screenshot."));
|
||||
} finally {
|
||||
setScreenshotLoading(false);
|
||||
}
|
||||
}, [getClient, revokeScreenshotUrl, screenshotFormat, screenshotQuality, screenshotFullPage, screenshotSelector]);
|
||||
|
||||
// Tabs
|
||||
const loadTabs = useCallback(async () => {
|
||||
setTabsLoading(true);
|
||||
setTabsError(null);
|
||||
try {
|
||||
const result = await getClient().getBrowserTabs();
|
||||
setTabs(result.tabs);
|
||||
} catch (err) {
|
||||
setTabsError(extractErrorMessage(err, "Unable to load tabs."));
|
||||
} finally {
|
||||
setTabsLoading(false);
|
||||
}
|
||||
}, [getClient]);
|
||||
|
||||
const handleCreateTab = async () => {
|
||||
setTabActing("new");
|
||||
setTabsError(null);
|
||||
try {
|
||||
await getClient().createBrowserTab(newTabUrl.trim() ? { url: newTabUrl.trim() } : {});
|
||||
setNewTabUrl("");
|
||||
await loadTabs();
|
||||
} catch (err) {
|
||||
setTabsError(extractErrorMessage(err, "Unable to create tab."));
|
||||
} finally {
|
||||
setTabActing(null);
|
||||
}
|
||||
};
|
||||
|
||||
const handleActivateTab = async (tabId: string) => {
|
||||
setTabActing(tabId);
|
||||
setTabsError(null);
|
||||
try {
|
||||
await getClient().activateBrowserTab(tabId);
|
||||
await loadTabs();
|
||||
} catch (err) {
|
||||
setTabsError(extractErrorMessage(err, "Unable to activate tab."));
|
||||
} finally {
|
||||
setTabActing(null);
|
||||
}
|
||||
};
|
||||
|
||||
const handleCloseTab = async (tabId: string) => {
|
||||
setTabActing(tabId);
|
||||
setTabsError(null);
|
||||
try {
|
||||
await getClient().closeBrowserTab(tabId);
|
||||
await loadTabs();
|
||||
} catch (err) {
|
||||
setTabsError(extractErrorMessage(err, "Unable to close tab."));
|
||||
} finally {
|
||||
setTabActing(null);
|
||||
}
|
||||
};
|
||||
|
||||
// Console
|
||||
const loadConsole = useCallback(async () => {
|
||||
setConsoleLoading(true);
|
||||
setConsoleError(null);
|
||||
try {
|
||||
const query = consoleLevel !== "all" ? { level: consoleLevel } : {};
|
||||
const result = await getClient().getBrowserConsole(query);
|
||||
setConsoleMessages(result.messages);
|
||||
} catch (err) {
|
||||
setConsoleError(extractErrorMessage(err, "Unable to load console messages."));
|
||||
} finally {
|
||||
setConsoleLoading(false);
|
||||
}
|
||||
}, [getClient, consoleLevel]);
|
||||
|
||||
// Initial load
|
||||
useEffect(() => {
|
||||
void loadStatus();
|
||||
|
|
@ -103,6 +264,26 @@ const BrowserTab = ({ getClient }: { getClient: () => SandboxAgent }) => {
|
|||
return () => clearInterval(interval);
|
||||
}, [status?.state, loadStatus]);
|
||||
|
||||
// Load tabs and console when browser becomes active
|
||||
useEffect(() => {
|
||||
if (status?.state === "active") {
|
||||
void loadTabs();
|
||||
void loadConsole();
|
||||
}
|
||||
}, [status?.state, loadTabs, loadConsole]);
|
||||
|
||||
// Auto-refresh console every 3s when active
|
||||
useEffect(() => {
|
||||
if (status?.state !== "active") return;
|
||||
const interval = setInterval(() => void loadConsole(), 3000);
|
||||
return () => clearInterval(interval);
|
||||
}, [status?.state, loadConsole]);
|
||||
|
||||
// Cleanup screenshot URL on unmount
|
||||
useEffect(() => {
|
||||
return () => revokeScreenshotUrl();
|
||||
}, [revokeScreenshotUrl]);
|
||||
|
||||
// Reset live view when browser becomes inactive
|
||||
useEffect(() => {
|
||||
if (status?.state !== "active") {
|
||||
|
|
@ -438,6 +619,253 @@ const BrowserTab = ({ getClient }: { getClient: () => SandboxAgent }) => {
|
|||
|
||||
{isActive && !liveViewActive && <div className="desktop-screenshot-empty">Click "Start Stream" for live browser view.</div>}
|
||||
</div>
|
||||
|
||||
{/* ========== Screenshot Section ========== */}
|
||||
{isActive && (
|
||||
<div className="card">
|
||||
<div className="card-header">
|
||||
<span className="card-title">
|
||||
<Camera size={14} style={{ marginRight: 6 }} />
|
||||
Screenshot
|
||||
</span>
|
||||
<button
|
||||
className="button secondary small"
|
||||
onClick={() => void refreshScreenshot()}
|
||||
disabled={screenshotLoading}
|
||||
style={{ padding: "4px 10px", fontSize: 11 }}
|
||||
>
|
||||
{screenshotLoading ? <Loader2 size={12} className="spinner-icon" /> : <Camera size={12} style={{ marginRight: 4 }} />}
|
||||
Capture
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="desktop-screenshot-controls">
|
||||
<div className="desktop-input-group">
|
||||
<label className="label">Format</label>
|
||||
<select className="setup-input mono" value={screenshotFormat} onChange={(e) => setScreenshotFormat(e.target.value as "png" | "jpeg" | "webp")}>
|
||||
<option value="png">PNG</option>
|
||||
<option value="jpeg">JPEG</option>
|
||||
<option value="webp">WebP</option>
|
||||
</select>
|
||||
</div>
|
||||
{screenshotFormat !== "png" && (
|
||||
<div className="desktop-input-group">
|
||||
<label className="label">Quality</label>
|
||||
<input
|
||||
className="setup-input mono"
|
||||
value={screenshotQuality}
|
||||
onChange={(e) => setScreenshotQuality(e.target.value)}
|
||||
inputMode="numeric"
|
||||
style={{ maxWidth: 60 }}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
<label className="desktop-checkbox-label">
|
||||
<input type="checkbox" checked={screenshotFullPage} onChange={(e) => setScreenshotFullPage(e.target.checked)} />
|
||||
Full page
|
||||
</label>
|
||||
<div className="desktop-input-group">
|
||||
<label className="label">Selector</label>
|
||||
<input
|
||||
className="setup-input mono"
|
||||
value={screenshotSelector}
|
||||
onChange={(e) => setScreenshotSelector(e.target.value)}
|
||||
placeholder="e.g. #main"
|
||||
style={{ maxWidth: 140 }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{screenshotError && (
|
||||
<div className="banner error" style={{ marginBottom: 8 }}>
|
||||
{screenshotError}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{screenshotUrl ? (
|
||||
<div className="desktop-screenshot-frame">
|
||||
<img src={screenshotUrl} alt="Browser screenshot" className="desktop-screenshot-image" />
|
||||
</div>
|
||||
) : (
|
||||
<div className="desktop-screenshot-empty">Click "Capture" to take a browser screenshot.</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* ========== Tabs Section ========== */}
|
||||
{isActive && (
|
||||
<div className="card">
|
||||
<div className="card-header">
|
||||
<span className="card-title">
|
||||
<Layers size={14} style={{ marginRight: 6 }} />
|
||||
Tabs
|
||||
</span>
|
||||
<button className="button secondary small" onClick={() => void loadTabs()} disabled={tabsLoading} style={{ padding: "4px 8px", fontSize: 11 }}>
|
||||
{tabsLoading ? <Loader2 size={12} className="spinner-icon" /> : <RefreshCw size={12} />}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{tabsError && (
|
||||
<div className="banner error" style={{ marginBottom: 8 }}>
|
||||
{tabsError}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{tabs.length > 0 ? (
|
||||
<div className="desktop-process-list">
|
||||
{tabs.map((tab) => (
|
||||
<div key={tab.id} className={`desktop-window-item ${tab.active ? "desktop-window-focused" : ""}`}>
|
||||
<div style={{ display: "flex", alignItems: "center", justifyContent: "space-between" }}>
|
||||
<div style={{ minWidth: 0, flex: 1 }}>
|
||||
<strong style={{ fontSize: 12 }}>{tab.title || "(untitled)"}</strong>
|
||||
{tab.active && (
|
||||
<span className="pill success" style={{ marginLeft: 8 }}>
|
||||
active
|
||||
</span>
|
||||
)}
|
||||
<div className="mono" style={{ fontSize: 11, color: "var(--muted)", marginTop: 2, wordBreak: "break-all" }}>
|
||||
{tab.url}
|
||||
</div>
|
||||
</div>
|
||||
<div style={{ display: "flex", gap: 4, flexShrink: 0, marginLeft: 8 }}>
|
||||
{!tab.active && (
|
||||
<button
|
||||
className="button ghost small"
|
||||
title="Activate"
|
||||
onClick={() => void handleActivateTab(tab.id)}
|
||||
disabled={tabActing === tab.id}
|
||||
style={{ padding: "4px 8px", fontSize: 11 }}
|
||||
>
|
||||
{tabActing === tab.id ? <Loader2 size={12} className="spinner-icon" /> : <Play size={12} />}
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
className="button ghost small"
|
||||
title="Close"
|
||||
onClick={() => void handleCloseTab(tab.id)}
|
||||
disabled={tabActing === tab.id}
|
||||
style={{ padding: "4px 8px", fontSize: 11 }}
|
||||
>
|
||||
<X size={12} />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div className="desktop-screenshot-empty">No tabs open.</div>
|
||||
)}
|
||||
|
||||
<div style={{ marginTop: 10, display: "flex", gap: 6, alignItems: "center" }}>
|
||||
<input
|
||||
className="setup-input mono"
|
||||
value={newTabUrl}
|
||||
onChange={(e) => setNewTabUrl(e.target.value)}
|
||||
placeholder="https://example.com"
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === "Enter") void handleCreateTab();
|
||||
}}
|
||||
style={{ flex: 1, fontSize: 11 }}
|
||||
/>
|
||||
<button
|
||||
className="button secondary small"
|
||||
onClick={() => void handleCreateTab()}
|
||||
disabled={tabActing === "new"}
|
||||
style={{ padding: "4px 10px", fontSize: 11 }}
|
||||
>
|
||||
{tabActing === "new" ? <Loader2 size={12} className="spinner-icon" /> : <Plus size={12} style={{ marginRight: 4 }} />}
|
||||
New Tab
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* ========== Console Section ========== */}
|
||||
{isActive && (
|
||||
<div className="card">
|
||||
<div className="card-header">
|
||||
<span className="card-title">
|
||||
<Terminal size={14} style={{ marginRight: 6 }} />
|
||||
Console
|
||||
</span>
|
||||
<div style={{ display: "flex", gap: 6, alignItems: "center" }}>
|
||||
<button
|
||||
className="button secondary small"
|
||||
onClick={() => void loadConsole()}
|
||||
disabled={consoleLoading}
|
||||
style={{ padding: "4px 8px", fontSize: 11 }}
|
||||
>
|
||||
{consoleLoading ? <Loader2 size={12} className="spinner-icon" /> : <RefreshCw size={12} />}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Level filter pills */}
|
||||
<div style={{ display: "flex", gap: 4, marginBottom: 8, flexWrap: "wrap" }}>
|
||||
{CONSOLE_LEVELS.map((level) => (
|
||||
<button
|
||||
key={level}
|
||||
className={`button small ${consoleLevel === level ? "secondary" : "ghost"}`}
|
||||
onClick={() => setConsoleLevel(level)}
|
||||
style={{ padding: "2px 10px", fontSize: 11, textTransform: "capitalize" }}
|
||||
>
|
||||
{level}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{consoleError && (
|
||||
<div className="banner error" style={{ marginBottom: 8 }}>
|
||||
{consoleError}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{consoleMessages.length > 0 ? (
|
||||
<div
|
||||
style={{
|
||||
maxHeight: 240,
|
||||
overflowY: "auto",
|
||||
border: "1px solid var(--border)",
|
||||
borderRadius: 6,
|
||||
padding: 6,
|
||||
background: "var(--background, #0f172a)",
|
||||
}}
|
||||
>
|
||||
{consoleMessages.map((msg, idx) => (
|
||||
<div
|
||||
key={`${msg.timestamp}-${idx}`}
|
||||
style={{
|
||||
display: "flex",
|
||||
gap: 8,
|
||||
padding: "3px 4px",
|
||||
fontSize: 11,
|
||||
fontFamily: "monospace",
|
||||
borderBottom: idx < consoleMessages.length - 1 ? "1px solid var(--border)" : undefined,
|
||||
}}
|
||||
>
|
||||
<span
|
||||
style={{
|
||||
width: 6,
|
||||
height: 6,
|
||||
borderRadius: "50%",
|
||||
background: consoleLevelColor(msg.level),
|
||||
flexShrink: 0,
|
||||
marginTop: 5,
|
||||
}}
|
||||
/>
|
||||
<span style={{ color: consoleLevelColor(msg.level), minWidth: 36, flexShrink: 0 }}>{msg.level}</span>
|
||||
<span style={{ flex: 1, wordBreak: "break-all", color: "var(--foreground, #e2e8f0)" }}>{msg.text}</span>
|
||||
<span style={{ color: "var(--muted)", flexShrink: 0, fontSize: 10 }}>{new Date(msg.timestamp).toLocaleTimeString()}</span>
|
||||
</div>
|
||||
))}
|
||||
<div ref={consoleEndRef} />
|
||||
</div>
|
||||
) : (
|
||||
<div className="desktop-screenshot-empty">No console messages.</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue