getcompanion-ai/co-mono
1
0
Fork 0
mirror of https://github.com/getcompanion-ai/co-mono.git synced 2026-04-15 15:03:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix various sandbox issues.

Browse source
This commit is contained in:
Mario Zechner 2025-10-08 22:51:32 +02:00
parent 91c1dc6475
commit 0eaa879d46
10 changed files with 673 additions and 431 deletions
Download patch file Download diff file Expand all files Collapse all files

49
packages/web-ui/src/ChatPanel.ts
View file

@ -1,12 +1,15 @@
import { Badge, html } from "@mariozechner/mini-lit";
import { LitElement } from "lit";
import { customElement, property, state } from "lit/decorators.js";
import type { AgentInterface } from "./components/AgentInterface.js";
import "./components/AgentInterface.js";
import type { Agent } from "./agent/agent.js";
import "./components/AgentInterface.js";
import type { AgentInterface } from "./components/AgentInterface.js";
import { AttachmentsRuntimeProvider } from "./components/sandbox/AttachmentsRuntimeProvider.js";
import type { SandboxRuntimeProvider } from "./components/sandbox/SandboxRuntimeProvider.js";
import { ArtifactsPanel, ArtifactsToolRenderer } from "./tools/artifacts/index.js";
import { createJavaScriptReplTool } from "./tools/javascript-repl.js";
import { registerToolRenderer } from "./tools/renderer-registry.js";
import type { Attachment } from "./utils/attachment-utils.js";
import { i18n } from "./utils/i18n.js";
const BREAKPOINT = 800; // px - switch between overlay and side-by-side
@ -20,6 +23,21 @@ export class ChatPanel extends LitElement {
@state() private artifactCount = 0;
@state() private showArtifactsPanel = false;
@state() private windowWidth = 0;
@property({ attribute: false }) runtimeProvidersFactory = () => {
const attachments: Attachment[] = [];
for (const message of this.agent!.state.messages) {
if (message.role === "user") {
message.attachments?.forEach((a) => {
attachments.push(a);
});
}
}
const providers: SandboxRuntimeProvider[] = [];
if (attachments.length > 0) {
providers.push(new AttachmentsRuntimeProvider(attachments));
}
return providers;
};
@property({ attribute: false }) sandboxUrlProvider?: () => string;
@property({ attribute: false }) onApiKeyRequired?: (provider: string) => Promise<boolean>;
@property({ attribute: false }) onBeforeSend?: () => void | Promise<void>;
@ -81,30 +99,9 @@ export class ChatPanel extends LitElement {
// Register the standalone tool renderer (not the panel itself)
registerToolRenderer("artifacts", new ArtifactsToolRenderer(this.artifactsPanel));
// Attachments provider
const getAttachments = () => {
const attachments: any[] = [];
for (const message of this.agent!.state.messages) {
if (message.role === "user") {
const content = Array.isArray(message.content) ? message.content : [message.content];
for (const block of content) {
if (typeof block !== "string" && block.type === "image") {
attachments.push({
id: `image-${attachments.length}`,
fileName: "image.png",
mimeType: block.mimeType || "image/png",
size: 0,
content: block.data,
});
}
}
}
}
return attachments;
};
javascriptReplTool.attachmentsProvider = getAttachments;
this.artifactsPanel.attachmentsProvider = getAttachments;
// Runtime providers factory
javascriptReplTool.runtimeProvidersFactory = this.runtimeProvidersFactory;
this.artifactsPanel.runtimeProvidersFactory = this.runtimeProvidersFactory;
this.artifactsPanel.onArtifactsChange = () => {
const count = this.artifactsPanel?.artifacts?.size ?? 0;

472
packages/web-ui/src/components/SandboxedIframe.ts
View file

@ -1,6 +1,8 @@
import { LitElement } from "lit";
import { customElement, property } from "lit/decorators.js";
import type { Attachment } from "../utils/attachment-utils.js";
import { ConsoleRuntimeProvider } from "./sandbox/ConsoleRuntimeProvider.js";
import { type MessageConsumer, SANDBOX_MESSAGE_ROUTER } from "./sandbox/SandboxMessageRouter.js";
import type { SandboxRuntimeProvider } from "./sandbox/SandboxRuntimeProvider.js";
export interface SandboxFile {
fileName: string;
@ -42,6 +44,9 @@ export class SandboxIframe extends LitElement {
override disconnectedCallback() {
super.disconnectedCallback();
// Note: We don't unregister the sandbox here for loadContent() mode
// because the caller (HtmlArtifact) owns the sandbox lifecycle.
// For execute() mode, the sandbox is unregistered in the cleanup function.
this.iframe?.remove();
}
@ -49,65 +54,88 @@ export class SandboxIframe extends LitElement {
* Load HTML content into sandbox and keep it displayed (for HTML artifacts)
* @param sandboxId Unique ID
* @param htmlContent Full HTML content
* @param attachments Attachments available
* @param providers Runtime providers to inject
* @param consumers Message consumers to register (optional)
*/
public loadContent(sandboxId: string, htmlContent: string, attachments: Attachment[]): void {
const completeHtml = this.prepareHtmlDocument(sandboxId, htmlContent, attachments);
public loadContent(
sandboxId: string,
htmlContent: string,
providers: SandboxRuntimeProvider[] = [],
consumers: MessageConsumer[] = [],
): void {
// Unregister previous sandbox if exists
try {
SANDBOX_MESSAGE_ROUTER.unregisterSandbox(sandboxId);
} catch {
// Sandbox might not exist, that's ok
}
providers = [new ConsoleRuntimeProvider(), ...providers];
SANDBOX_MESSAGE_ROUTER.registerSandbox(sandboxId, providers, consumers);
const completeHtml = this.prepareHtmlDocument(sandboxId, htmlContent, providers);
// Remove previous iframe if exists
this.iframe?.remove();
if (this.sandboxUrlProvider) {
// Browser extension mode: use sandbox.html with postMessage
this.loadViaSandboxUrl(sandboxId, completeHtml, attachments);
this.loadViaSandboxUrl(sandboxId, completeHtml);
} else {
// Web mode: use srcdoc
this.loadViaSrcdoc(completeHtml);
this.loadViaSrcdoc(sandboxId, completeHtml);
}
}
private loadViaSandboxUrl(sandboxId: string, completeHtml: string, attachments: Attachment[]): void {
// Wait for sandbox-ready and send content
private loadViaSandboxUrl(sandboxId: string, completeHtml: string): void {
// Create iframe pointing to sandbox URL
this.iframe = document.createElement("iframe");
this.iframe.sandbox.add("allow-scripts");
this.iframe.sandbox.add("allow-modals");
this.iframe.style.width = "100%";
this.iframe.style.height = "100%";
this.iframe.style.border = "none";
this.iframe.src = this.sandboxUrlProvider!();
// Update router with iframe reference BEFORE appending to DOM
SANDBOX_MESSAGE_ROUTER.setSandboxIframe(sandboxId, this.iframe);
// Listen for sandbox-ready message directly
const readyHandler = (e: MessageEvent) => {
if (e.data.type === "sandbox-ready" && e.source === this.iframe?.contentWindow) {
window.removeEventListener("message", readyHandler);
// Send content to sandbox
this.iframe?.contentWindow?.postMessage(
{
type: "sandbox-load",
sandboxId,
code: completeHtml,
attachments,
},
"*",
);
}
};
window.addEventListener("message", readyHandler);
// Always recreate iframe to ensure fresh sandbox and sandbox-ready message
this.iframe?.remove();
this.iframe = document.createElement("iframe");
this.iframe.sandbox.add("allow-scripts");
this.iframe.sandbox.add("allow-modals");
this.iframe.style.width = "100%";
this.iframe.style.height = "100%";
this.iframe.style.border = "none";
this.iframe.src = this.sandboxUrlProvider!();
this.appendChild(this.iframe);
}
private loadViaSrcdoc(completeHtml: string): void {
// Always recreate iframe to ensure fresh sandbox
this.iframe?.remove();
private loadViaSrcdoc(sandboxId: string, completeHtml: string): void {
// Create iframe with srcdoc
this.iframe = document.createElement("iframe");
this.iframe.sandbox.add("allow-scripts");
this.iframe.sandbox.add("allow-modals");
this.iframe.style.width = "100%";
this.iframe.style.height = "100%";
this.iframe.style.border = "none";
// Set content directly via srcdoc (no CSP restrictions in web apps)
this.iframe.srcdoc = completeHtml;
// Update router with iframe reference BEFORE appending to DOM
SANDBOX_MESSAGE_ROUTER.setSandboxIframe(sandboxId, this.iframe);
this.appendChild(this.iframe);
}
@ -115,143 +143,141 @@ export class SandboxIframe extends LitElement {
* Execute code in sandbox
* @param sandboxId Unique ID for this execution
* @param code User code (plain JS for REPL, or full HTML for artifacts)
* @param attachments Attachments available to the code
* @param providers Runtime providers to inject
* @param consumers Additional message consumers (optional, execute has its own internal consumer)
* @param signal Abort signal
* @returns Promise resolving to execution result
*/
public async execute(
sandboxId: string,
code: string,
attachments: Attachment[],
providers: SandboxRuntimeProvider[] = [],
consumers: MessageConsumer[] = [],
signal?: AbortSignal,
): Promise<SandboxResult> {
if (signal?.aborted) {
throw new Error("Execution aborted");
}
// Prepare the complete HTML document with runtime + user code
const completeHtml = this.prepareHtmlDocument(sandboxId, code, attachments);
providers = [new ConsoleRuntimeProvider(), ...providers];
SANDBOX_MESSAGE_ROUTER.registerSandbox(sandboxId, providers, consumers);
const logs: Array<{ type: string; text: string }> = [];
const files: SandboxFile[] = [];
let completed = false;
// Wait for execution to complete
return new Promise((resolve, reject) => {
const logs: Array<{ type: string; text: string }> = [];
const files: SandboxFile[] = [];
let completed = false;
const messageHandler = (e: MessageEvent) => {
// Ignore messages not for this sandbox
if (e.data.sandboxId !== sandboxId) return;
if (e.data.type === "console") {
logs.push({
type: e.data.method === "error" ? "error" : "log",
text: e.data.text,
});
} else if (e.data.type === "file-returned") {
files.push({
fileName: e.data.fileName,
content: e.data.content,
mimeType: e.data.mimeType,
});
} else if (e.data.type === "execution-complete") {
completed = true;
cleanup();
resolve({
success: true,
console: logs,
files: files,
});
} else if (e.data.type === "execution-error") {
completed = true;
cleanup();
resolve({
success: false,
console: logs,
error: e.data.error,
files,
});
}
// 4. Create execution consumer for lifecycle messages
const executionConsumer: MessageConsumer = {
handleMessage(message: any): boolean {
if (message.type === "console") {
logs.push({
type: message.method === "error" ? "error" : "log",
text: message.text,
});
return true;
} else if (message.type === "file-returned") {
files.push({
fileName: message.fileName,
content: message.content,
mimeType: message.mimeType,
});
return true;
} else if (message.type === "execution-complete") {
completed = true;
cleanup();
resolve({ success: true, console: logs, files });
return true;
} else if (message.type === "execution-error") {
completed = true;
cleanup();
resolve({ success: false, console: logs, error: message.error, files });
return true;
}
return false;
},
};
SANDBOX_MESSAGE_ROUTER.addConsumer(sandboxId, executionConsumer);
const cleanup = () => {
SANDBOX_MESSAGE_ROUTER.unregisterSandbox(sandboxId);
signal?.removeEventListener("abort", abortHandler);
clearTimeout(timeoutId);
this.iframe?.remove();
this.iframe = undefined;
};
// Abort handler
const abortHandler = () => {
if (!completed) {
completed = true;
cleanup();
reject(new Error("Execution aborted"));
}
};
let readyHandler: ((e: MessageEvent) => void) | undefined;
if (signal) {
signal.addEventListener("abort", abortHandler);
}
const cleanup = () => {
window.removeEventListener("message", messageHandler);
signal?.removeEventListener("abort", abortHandler);
if (readyHandler) {
window.removeEventListener("message", readyHandler);
}
clearTimeout(timeoutId);
};
// Set up listeners BEFORE creating iframe
window.addEventListener("message", messageHandler);
signal?.addEventListener("abort", abortHandler);
// Timeout after 30 seconds
// Timeout handler (30 seconds)
const timeoutId = setTimeout(() => {
if (!completed) {
completed = true;
cleanup();
resolve({
success: false,
error: { message: "Execution timeout (30s)", stack: "" },
console: logs,
error: { message: "Execution timeout (30s)", stack: "" },
files,
});
}
}, 30000);
// 4. Prepare HTML and create iframe
const completeHtml = this.prepareHtmlDocument(sandboxId, code, providers);
if (this.sandboxUrlProvider) {
// Browser extension mode: wait for sandbox-ready and send content
readyHandler = (e: MessageEvent) => {
// Browser extension mode: wait for sandbox-ready
this.iframe = document.createElement("iframe");
this.iframe.sandbox.add("allow-scripts", "allow-modals");
this.iframe.style.cssText = "width: 100%; height: 100%; border: none;";
this.iframe.src = this.sandboxUrlProvider();
// Update router with iframe reference BEFORE appending to DOM
SANDBOX_MESSAGE_ROUTER.setSandboxIframe(sandboxId, this.iframe);
// Listen for sandbox-ready message directly
const readyHandler = (e: MessageEvent) => {
if (e.data.type === "sandbox-ready" && e.source === this.iframe?.contentWindow) {
window.removeEventListener("message", readyHandler!);
// Send the complete HTML
window.removeEventListener("message", readyHandler);
// Send content to sandbox
this.iframe?.contentWindow?.postMessage(
{
type: "sandbox-load",
sandboxId,
code: completeHtml,
attachments,
},
"*",
);
}
};
window.addEventListener("message", readyHandler);
// Create iframe AFTER all listeners are set up
this.iframe?.remove();
this.iframe = document.createElement("iframe");
this.iframe.sandbox.add("allow-scripts");
this.iframe.sandbox.add("allow-modals");
this.iframe.style.width = "100%";
this.iframe.style.height = "100%";
this.iframe.style.border = "none";
this.iframe.src = this.sandboxUrlProvider();
this.appendChild(this.iframe);
} else {
// Web mode: use srcdoc
this.iframe?.remove();
this.iframe = document.createElement("iframe");
this.iframe.sandbox.add("allow-scripts");
this.iframe.sandbox.add("allow-modals");
this.iframe.style.width = "100%";
this.iframe.style.height = "100%";
this.iframe.style.border = "none";
// Set content via srcdoc BEFORE appending to DOM
this.iframe.sandbox.add("allow-scripts", "allow-modals");
this.iframe.style.cssText = "width: 100%; height: 100%; border: none; display: none;";
this.iframe.srcdoc = completeHtml;
// Update router with iframe reference BEFORE appending to DOM
SANDBOX_MESSAGE_ROUTER.setSandboxIframe(sandboxId, this.iframe);
this.appendChild(this.iframe);
}
});
@ -259,10 +285,11 @@ export class SandboxIframe extends LitElement {
/**
* Prepare complete HTML document with runtime + user code
* PUBLIC so HtmlArtifact can use it for download button
*/
private prepareHtmlDocument(sandboxId: string, userCode: string, attachments: Attachment[]): string {
public prepareHtmlDocument(sandboxId: string, userCode: string, providers: SandboxRuntimeProvider[] = []): string {
// Runtime script that will be injected
const runtime = this.getRuntimeScript(sandboxId, attachments);
const runtime = this.getRuntimeScript(sandboxId, providers);
// Check if user provided full HTML
const hasHtmlTag = /<html[^>]*>/i.test(userCode);
@ -311,215 +338,30 @@ export class SandboxIframe extends LitElement {
}
/**
* Get the runtime script that captures console, provides helpers, etc.
* Generate runtime script from providers
*/
private getRuntimeScript(sandboxId: string, attachments: Attachment[]): string {
// Convert attachments to serializable format
const attachmentsData = attachments.map((a) => ({
id: a.id,
fileName: a.fileName,
mimeType: a.mimeType,
size: a.size,
content: a.content,
extractedText: a.extractedText,
}));
private getRuntimeScript(sandboxId: string, providers: SandboxRuntimeProvider[] = []): string {
// Collect all data from providers
const allData: Record<string, any> = {};
for (const provider of providers) {
Object.assign(allData, provider.getData());
}
// Runtime function that will run in the sandbox (NO parameters - values injected before function)
const runtimeFunc = () => {
// Helper functions
(window as any).listFiles = () =>
(attachments || []).map((a: any) => ({
id: a.id,
fileName: a.fileName,
mimeType: a.mimeType,
size: a.size,
}));
// Collect all runtime functions - pass sandboxId as string literal
const runtimeFunctions: string[] = [];
for (const provider of providers) {
runtimeFunctions.push(`(${provider.getRuntime().toString()})(${JSON.stringify(sandboxId)});`);
}
(window as any).readTextFile = (attachmentId: string) => {
const a = (attachments || []).find((x: any) => x.id === attachmentId);
if (!a) throw new Error("Attachment not found: " + attachmentId);
if (a.extractedText) return a.extractedText;
try {
return atob(a.content);
} catch {
throw new Error("Failed to decode text content for: " + attachmentId);
}
};
// Build script
const dataInjection = Object.entries(allData)
.map(([key, value]) => `window.${key} = ${JSON.stringify(value)};`)
.join("\n");
(window as any).readBinaryFile = (attachmentId: string) => {
const a = (attachments || []).find((x: any) => x.id === attachmentId);
if (!a) throw new Error("Attachment not found: " + attachmentId);
const bin = atob(a.content);
const bytes = new Uint8Array(bin.length);
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
return bytes;
};
(window as any).returnFile = async (fileName: string, content: any, mimeType?: string) => {
let finalContent: any, finalMimeType: string;
if (content instanceof Blob) {
const arrayBuffer = await content.arrayBuffer();
finalContent = new Uint8Array(arrayBuffer);
finalMimeType = mimeType || content.type || "application/octet-stream";
if (!mimeType && !content.type) {
throw new Error(
"returnFile: MIME type is required for Blob content. Please provide a mimeType parameter (e.g., 'image/png').",
);
}
} else if (content instanceof Uint8Array) {
finalContent = content;
if (!mimeType) {
throw new Error(
"returnFile: MIME type is required for Uint8Array content. Please provide a mimeType parameter (e.g., 'image/png').",
);
}
finalMimeType = mimeType;
} else if (typeof content === "string") {
finalContent = content;
finalMimeType = mimeType || "text/plain";
} else {
finalContent = JSON.stringify(content, null, 2);
finalMimeType = mimeType || "application/json";
}
window.parent.postMessage(
{
type: "file-returned",
sandboxId,
fileName,
content: finalContent,
mimeType: finalMimeType,
},
"*",
);
};
// Console capture
const originalConsole = {
log: console.log,
error: console.error,
warn: console.warn,
info: console.info,
};
["log", "error", "warn", "info"].forEach((method) => {
(console as any)[method] = (...args: any[]) => {
const text = args
.map((arg) => {
try {
return typeof arg === "object" ? JSON.stringify(arg) : String(arg);
} catch {
return String(arg);
}
})
.join(" ");
window.parent.postMessage(
{
type: "console",
sandboxId,
method,
text,
},
"*",
);
(originalConsole as any)[method].apply(console, args);
};
});
// Track errors for HTML artifacts
let lastError: { message: string; stack: string } | null = null;
// Error handlers
window.addEventListener("error", (e) => {
const text =
(e.error?.stack || e.message || String(e)) + " at line " + (e.lineno || "?") + ":" + (e.colno || "?");
// Store the error
lastError = {
message: e.error?.message || e.message || String(e),
stack: e.error?.stack || text,
};
window.parent.postMessage(
{
type: "console",
sandboxId,
method: "error",
text,
},
"*",
);
});
window.addEventListener("unhandledrejection", (e) => {
const text = "Unhandled promise rejection: " + (e.reason?.message || e.reason || "Unknown error");
// Store the error
lastError = {
message: e.reason?.message || String(e.reason) || "Unhandled promise rejection",
stack: e.reason?.stack || text,
};
window.parent.postMessage(
{
type: "console",
sandboxId,
method: "error",
text,
},
"*",
);
});
// Expose complete() method for user code to call
let completionSent = false;
(window as any).complete = (error?: { message: string; stack: string }) => {
if (completionSent) return;
completionSent = true;
// Use provided error or last caught error
const finalError = error || lastError;
if (finalError) {
window.parent.postMessage(
{
type: "execution-error",
sandboxId,
error: finalError,
},
"*",
);
} else {
window.parent.postMessage(
{
type: "execution-complete",
sandboxId,
},
"*",
);
}
};
// Fallback timeout for HTML artifacts that don't call complete()
if (document.readyState === "complete" || document.readyState === "interactive") {
setTimeout(() => (window as any).complete(), 2000);
} else {
window.addEventListener("load", () => {
setTimeout(() => (window as any).complete(), 2000);
});
}
};
// Prepend the const declarations, then the function
return (
`<script>\n` +
`window.sandboxId = ${JSON.stringify(sandboxId)};\n` +
`window.attachments = ${JSON.stringify(attachmentsData)};\n` +
`(${runtimeFunc.toString()})();\n` +
`</script>`
);
return `<script>
window.sandboxId = ${JSON.stringify(sandboxId)};
${dataInjection}
${runtimeFunctions.join("\n")}
</script>`;
}
}

99
packages/web-ui/src/components/sandbox/AttachmentsRuntimeProvider.ts Normal file
View file

@ -0,0 +1,99 @@
import type { Attachment } from "../../utils/attachment-utils.js";
import type { SandboxRuntimeProvider } from "./SandboxRuntimeProvider.js";
/**
* Attachments Runtime Provider
*
* OPTIONAL provider that provides file access APIs to sandboxed code.
* Only needed when attachments are present.
*/
export class AttachmentsRuntimeProvider implements SandboxRuntimeProvider {
constructor(private attachments: Attachment[]) {}
getData(): Record<string, any> {
const attachmentsData = this.attachments.map((a) => ({
id: a.id,
fileName: a.fileName,
mimeType: a.mimeType,
size: a.size,
content: a.content,
extractedText: a.extractedText,
}));
return { attachments: attachmentsData };
}
getRuntime(): (sandboxId: string) => void {
// This function will be stringified, so no external references!
return (sandboxId: string) => {
// Helper functions for attachments
(window as any).listFiles = () =>
((window as any).attachments || []).map((a: any) => ({
id: a.id,
fileName: a.fileName,
mimeType: a.mimeType,
size: a.size,
}));
(window as any).readTextFile = (attachmentId: string) => {
const a = ((window as any).attachments || []).find((x: any) => x.id === attachmentId);
if (!a) throw new Error("Attachment not found: " + attachmentId);
if (a.extractedText) return a.extractedText;
try {
return atob(a.content);
} catch {
throw new Error("Failed to decode text content for: " + attachmentId);
}
};
(window as any).readBinaryFile = (attachmentId: string) => {
const a = ((window as any).attachments || []).find((x: any) => x.id === attachmentId);
if (!a) throw new Error("Attachment not found: " + attachmentId);
const bin = atob(a.content);
const bytes = new Uint8Array(bin.length);
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
return bytes;
};
(window as any).returnFile = async (fileName: string, content: any, mimeType?: string) => {
let finalContent: any, finalMimeType: string;
if (content instanceof Blob) {
const arrayBuffer = await content.arrayBuffer();
finalContent = new Uint8Array(arrayBuffer);
finalMimeType = mimeType || content.type || "application/octet-stream";
if (!mimeType && !content.type) {
throw new Error(
"returnFile: MIME type is required for Blob content. Please provide a mimeType parameter (e.g., 'image/png').",
);
}
} else if (content instanceof Uint8Array) {
finalContent = content;
if (!mimeType) {
throw new Error(
"returnFile: MIME type is required for Uint8Array content. Please provide a mimeType parameter (e.g., 'image/png').",
);
}
finalMimeType = mimeType;
} else if (typeof content === "string") {
finalContent = content;
finalMimeType = mimeType || "text/plain";
} else {
finalContent = JSON.stringify(content, null, 2);
finalMimeType = mimeType || "application/json";
}
window.parent.postMessage(
{
type: "file-returned",
sandboxId,
fileName,
content: finalContent,
mimeType: finalMimeType,
},
"*",
);
};
};
}
}

132
packages/web-ui/src/components/sandbox/ConsoleRuntimeProvider.ts Normal file
View file

@ -0,0 +1,132 @@
import type { SandboxRuntimeProvider } from "./SandboxRuntimeProvider.js";
/**
* Console Runtime Provider
*
* REQUIRED provider that should always be included first.
* Provides console capture, error handling, and execution lifecycle management.
*/
export class ConsoleRuntimeProvider implements SandboxRuntimeProvider {
getData(): Record<string, any> {
// No data needed
return {};
}
getRuntime(): (sandboxId: string) => void {
return (sandboxId: string) => {
// Console capture
const originalConsole = {
log: console.log,
error: console.error,
warn: console.warn,
info: console.info,
};
["log", "error", "warn", "info"].forEach((method) => {
(console as any)[method] = (...args: any[]) => {
const text = args
.map((arg) => {
try {
return typeof arg === "object" ? JSON.stringify(arg) : String(arg);
} catch {
return String(arg);
}
})
.join(" ");
window.parent.postMessage(
{
type: "console",
sandboxId,
method,
text,
},
"*",
);
(originalConsole as any)[method].apply(console, args);
};
});
// Track errors for HTML artifacts
let lastError: { message: string; stack: string } | null = null;
// Error handlers
window.addEventListener("error", (e) => {
const text =
(e.error?.stack || e.message || String(e)) + " at line " + (e.lineno || "?") + ":" + (e.colno || "?");
lastError = {
message: e.error?.message || e.message || String(e),
stack: e.error?.stack || text,
};
window.parent.postMessage(
{
type: "console",
sandboxId,
method: "error",
text,
},
"*",
);
});
window.addEventListener("unhandledrejection", (e) => {
const text = "Unhandled promise rejection: " + (e.reason?.message || e.reason || "Unknown error");
lastError = {
message: e.reason?.message || String(e.reason) || "Unhandled promise rejection",
stack: e.reason?.stack || text,
};
window.parent.postMessage(
{
type: "console",
sandboxId,
method: "error",
text,
},
"*",
);
});
// Expose complete() method for user code to call
let completionSent = false;
(window as any).complete = (error?: { message: string; stack: string }) => {
if (completionSent) return;
completionSent = true;
const finalError = error || lastError;
if (finalError) {
window.parent.postMessage(
{
type: "execution-error",
sandboxId,
error: finalError,
},
"*",
);
} else {
window.parent.postMessage(
{
type: "execution-complete",
sandboxId,
},
"*",
);
}
};
// Fallback timeout for HTML artifacts that don't call complete()
if (document.readyState === "complete" || document.readyState === "interactive") {
setTimeout(() => (window as any).complete(), 2000);
} else {
window.addEventListener("load", () => {
setTimeout(() => (window as any).complete(), 2000);
});
}
};
}
}

152
packages/web-ui/src/components/sandbox/SandboxMessageRouter.ts Normal file
View file

@ -0,0 +1,152 @@
import type { SandboxRuntimeProvider } from "./SandboxRuntimeProvider.js";
/**
* Message consumer interface - components that want to receive messages from sandboxes
*/
export interface MessageConsumer {
/**
* Handle a message from a sandbox.
* @returns true if message was consumed (stops propagation), false otherwise
*/
handleMessage(message: any): boolean;
}
/**
* Sandbox context - tracks active sandboxes and their consumers
*/
interface SandboxContext {
sandboxId: string;
iframe: HTMLIFrameElement | null; // null until setSandboxIframe()
providers: SandboxRuntimeProvider[];
consumers: Set<MessageConsumer>;
}
/**
* Centralized message router for all sandbox communication.
*
* This singleton replaces all individual window.addEventListener("message") calls
* with a single global listener that routes messages to the appropriate handlers.
*
* Benefits:
* - Single global listener instead of multiple independent listeners
* - Automatic cleanup when sandboxes are destroyed
* - Support for bidirectional communication (providers) and broadcasting (consumers)
* - Clear lifecycle management
*/
export class SandboxMessageRouter {
private sandboxes = new Map<string, SandboxContext>();
private messageListener: ((e: MessageEvent) => void) | null = null;
/**
* Register a new sandbox with its runtime providers.
* Call this BEFORE creating the iframe.
*/
registerSandbox(sandboxId: string, providers: SandboxRuntimeProvider[], consumers: MessageConsumer[]): void {
this.sandboxes.set(sandboxId, {
sandboxId,
iframe: null, // Will be set via setSandboxIframe()
providers,
consumers: new Set(consumers),
});
// Setup global listener if not already done
this.setupListener();
console.log("Registered sandbox:", sandboxId);
}
/**
* Update the iframe reference for a sandbox.
* Call this AFTER creating the iframe.
* This is needed so providers can send responses back to the sandbox.
*/
setSandboxIframe(sandboxId: string, iframe: HTMLIFrameElement): void {
const context = this.sandboxes.get(sandboxId);
if (context) {
context.iframe = iframe;
}
console.log("Set iframe for sandbox:", sandboxId);
}
/**
* Unregister a sandbox and remove all its consumers.
* Call this when the sandbox is destroyed.
*/
unregisterSandbox(sandboxId: string): void {
this.sandboxes.delete(sandboxId);
// If no more sandboxes, remove global listener
if (this.sandboxes.size === 0 && this.messageListener) {
window.removeEventListener("message", this.messageListener);
this.messageListener = null;
}
console.log("Unregistered sandbox:", sandboxId);
}
/**
* Add a message consumer for a sandbox.
* Consumers receive broadcast messages (console, execution-complete, etc.)
*/
addConsumer(sandboxId: string, consumer: MessageConsumer): void {
const context = this.sandboxes.get(sandboxId);
if (context) {
context.consumers.add(consumer);
}
console.log("Added consumer for sandbox:", sandboxId);
}
/**
* Remove a message consumer from a sandbox.
*/
removeConsumer(sandboxId: string, consumer: MessageConsumer): void {
const context = this.sandboxes.get(sandboxId);
if (context) {
context.consumers.delete(consumer);
}
console.log("Removed consumer for sandbox:", sandboxId);
}
/**
* Setup the global message listener (called automatically)
*/
private setupListener(): void {
if (this.messageListener) return;
this.messageListener = (e: MessageEvent) => {
const { sandboxId } = e.data;
if (!sandboxId) return;
console.log("Router received message for sandbox:", sandboxId, e.data);
const context = this.sandboxes.get(sandboxId);
if (!context) return;
// Create respond() function for bidirectional communication
const respond = (response: any) => {
if (!response.sandboxId) response.sandboxId = sandboxId;
context.iframe?.contentWindow?.postMessage(response, "*");
};
// 1. Try provider handlers first (for bidirectional comm like memory)
for (const provider of context.providers) {
if (provider.handleMessage) {
const handled = provider.handleMessage(e.data, respond);
if (handled) return; // Stop if handled
}
}
// 2. Broadcast to consumers (for one-way messages like console)
for (const consumer of context.consumers) {
const consumed = consumer.handleMessage(e.data);
if (consumed) break; // Stop if consumed
}
};
window.addEventListener("message", this.messageListener);
}
}
/**
* Global singleton instance.
* Import this from wherever you need to interact with the message router.
*/
export const SANDBOX_MESSAGE_ROUTER = new SandboxMessageRouter();

30
packages/web-ui/src/components/sandbox/SandboxRuntimeProvider.ts Normal file
View file

@ -0,0 +1,30 @@
/**
* Interface for providing runtime capabilities to sandboxed iframes.
* Each provider injects data and runtime functions into the sandbox context.
*/
export interface SandboxRuntimeProvider {
/**
* Returns data to inject into window scope.
* Keys become window properties (e.g., { attachments: [...] } -> window.attachments)
*/
getData(): Record<string, any>;
/**
* Returns a runtime function that will be stringified and executed in the sandbox.
* The function receives sandboxId and has access to data from getData() via window.
*
* IMPORTANT: This function will be converted to string via .toString() and injected
* into the sandbox, so it cannot reference external variables or imports.
*/
getRuntime(): (sandboxId: string) => void;
/**
* Optional message handler for bidirectional communication.
* Return true if the message was handled, false to let other handlers try.
*
* @param message - The message from the sandbox
* @param respond - Function to send a response back to the sandbox
* @returns true if message was handled, false otherwise
*/
handleMessage?(message: any, respond: (response: any) => void): boolean;
}

66
packages/web-ui/src/tools/artifacts/HtmlArtifact.ts
View file

@ -5,7 +5,8 @@ import { customElement, property, state } from "lit/decorators.js";
import { createRef, type Ref, ref } from "lit/directives/ref.js";
import { unsafeHTML } from "lit/directives/unsafe-html.js";
import type { SandboxIframe } from "../../components/SandboxedIframe.js";
import type { Attachment } from "../../utils/attachment-utils.js";
import { type MessageConsumer, SANDBOX_MESSAGE_ROUTER } from "../../components/sandbox/SandboxMessageRouter.js";
import type { SandboxRuntimeProvider } from "../../components/sandbox/SandboxRuntimeProvider.js";
import { i18n } from "../../utils/i18n.js";
import "../../components/SandboxedIframe.js";
import { ArtifactElement } from "./ArtifactElement.js";
@ -16,7 +17,7 @@ import "./Console.js";
export class HtmlArtifact extends ArtifactElement {
@property() override filename = "";
@property({ attribute: false }) override displayTitle = "";
@property({ attribute: false }) attachments: Attachment[] = [];
@property({ attribute: false }) runtimeProviders: SandboxRuntimeProvider[] = [];
@property({ attribute: false }) sandboxUrlProvider?: () => string;
private _content = "";
@ -26,9 +27,6 @@ export class HtmlArtifact extends ArtifactElement {
private sandboxIframeRef: Ref<SandboxIframe> = createRef();
private consoleRef: Ref<Console> = createRef();
// Store message handler so we can remove it
private messageHandler?: (e: MessageEvent) => void;
@state() private viewMode: "preview" | "code" = "preview";
private setViewMode(mode: "preview" | "code") {
@ -47,11 +45,17 @@ export class HtmlArtifact extends ArtifactElement {
copyButton.title = i18n("Copy HTML");
copyButton.showText = false;
// Generate standalone HTML with all runtime code injected for download
const sandbox = this.sandboxIframeRef.value;
const sandboxId = `artifact-${this.filename}`;
const downloadContent =
sandbox?.prepareHtmlDocument(sandboxId, this._content, this.runtimeProviders || []) || this._content;
return html`
<div class="flex items-center gap-2">
${toggle}
${copyButton}
${DownloadButton({ content: this._content, filename: this.filename, mimeType: "text/html", title: i18n("Download HTML") })}
${DownloadButton({ content: downloadContent, filename: this.filename, mimeType: "text/html", title: i18n("Download HTML") })}
</div>
`;
}
@ -79,33 +83,29 @@ export class HtmlArtifact extends ArtifactElement {
sandbox.sandboxUrlProvider = this.sandboxUrlProvider;
}
// Remove previous message handler if it exists
if (this.messageHandler) {
window.removeEventListener("message", this.messageHandler);
}
const sandboxId = `artifact-${this.filename}`;
// Set up message listener to collect logs
this.messageHandler = (e: MessageEvent) => {
if (e.data.sandboxId !== sandboxId) return;
if (e.data.type === "console") {
// Create new array reference for Lit reactivity
this.logs = [
...this.logs,
{
type: e.data.method === "error" ? "error" : "log",
text: e.data.text,
},
];
this.requestUpdate(); // Re-render to show console
}
// Create consumer for console messages
const consumer: MessageConsumer = {
handleMessage: (message: any): boolean => {
if (message.type === "console") {
// Create new array reference for Lit reactivity
this.logs = [
...this.logs,
{
type: message.method === "error" ? "error" : "log",
text: message.text,
},
];
this.requestUpdate(); // Re-render to show console
return true;
}
return false;
},
};
window.addEventListener("message", this.messageHandler);
// Load content (iframe persists, doesn't get removed)
sandbox.loadContent(sandboxId, html, this.attachments);
// Load content - this handles sandbox registration, consumer registration, and iframe creation
sandbox.loadContent(sandboxId, html, this.runtimeProviders, [consumer]);
}
override get content(): string {
@ -114,11 +114,9 @@ export class HtmlArtifact extends ArtifactElement {
override disconnectedCallback() {
super.disconnectedCallback();
// Clean up message handler when element is removed from DOM
if (this.messageHandler) {
window.removeEventListener("message", this.messageHandler);
this.messageHandler = undefined;
}
// Unregister sandbox when element is removed from DOM
const sandboxId = `artifact-${this.filename}`;
SANDBOX_MESSAGE_ROUTER.unregisterSandbox(sandboxId);
}
override firstUpdated() {

24
packages/web-ui/src/tools/artifacts/artifacts-tool-renderer.ts
View file

@ -93,7 +93,11 @@ export class ArtifactsToolRenderer implements ToolRenderer<ArtifactsParams, unde
// For create/update/rewrite errors, show code block + console/error
if (command === "create" || command === "update" || command === "rewrite") {
const content = command === "update" ? params?.new_str || params?.old_str || "" : params?.content || "";
const content = params?.content || "";
const { old_str, new_str } = params || {};
const isDiff = command === "update";
const diffContent =
old_str !== undefined && new_str !== undefined ? Diff({ oldText: old_str, newText: new_str }) : "";
const isHtml = filename?.endsWith(".html");
@ -101,7 +105,7 @@ export class ArtifactsToolRenderer implements ToolRenderer<ArtifactsParams, unde
<div>
${renderCollapsibleHeader(state, FileCode2, renderHeaderWithPill(headerText, filename), contentRef, chevronRef, false)}
<div ${ref(contentRef)} class="max-h-0 overflow-hidden transition-all duration-300 space-y-3">
${content ? html`<code-block .code=${content} language=${getLanguageFromFilename(filename)}></code-block>` : ""}
${isDiff ? diffContent : content ? html`<code-block .code=${content} language=${getLanguageFromFilename(filename)}></code-block>` : ""}
${
isHtml
? html`<console-block .content=${result.output || i18n("An error occurred")} variant="error"></console-block>`
@ -156,7 +160,7 @@ export class ArtifactsToolRenderer implements ToolRenderer<ArtifactsParams, unde
}
// CREATE/UPDATE/REWRITE: always show code block, + console block for .html files
if (command === "create" || command === "update" || command === "rewrite") {
if (command === "create" || command === "rewrite") {
const codeContent = content || "";
const isHtml = filename?.endsWith(".html");
const logs = result.output || "";
@ -172,6 +176,20 @@ export class ArtifactsToolRenderer implements ToolRenderer<ArtifactsParams, unde
`;
}
if (command === "update") {
const isHtml = filename?.endsWith(".html");
const logs = result.output || "";
return html`
<div>
${renderCollapsibleHeader(state, FileCode2, renderHeaderWithPill(headerText, filename), contentRef, chevronRef, false)}
<div ${ref(contentRef)} class="max-h-0 overflow-hidden transition-all duration-300 space-y-3">
${Diff({ oldText: params.old_str || "", newText: params.new_str || "" })}
${isHtml && logs ? html`<console-block .content=${logs}></console-block>` : ""}
</div>
</div>
`;
}
// For DELETE, just show header
return html`
<div class="space-y-3">

57
packages/web-ui/src/tools/artifacts/artifacts.ts
View file

@ -6,7 +6,7 @@ import { html, LitElement, type TemplateResult } from "lit";
import { customElement, property, state } from "lit/decorators.js";
import { createRef, type Ref, ref } from "lit/directives/ref.js";
import { X } from "lucide";
import type { Attachment } from "../../utils/attachment-utils.js";
import type { SandboxRuntimeProvider } from "../../components/sandbox/SandboxRuntimeProvider.js";
import { i18n } from "../../utils/i18n.js";
import type { ArtifactElement } from "./ArtifactElement.js";
import { HtmlArtifact } from "./HtmlArtifact.js";
@ -45,8 +45,8 @@ export class ArtifactsPanel extends LitElement {
private artifactElements = new Map<string, ArtifactElement>();
private contentRef: Ref<HTMLDivElement> = createRef();
// External provider for attachments (decouples panel from AgentInterface)
@property({ attribute: false }) attachmentsProvider?: () => Attachment[];
// External factory for runtime providers (decouples panel from AgentInterface)
@property({ attribute: false }) runtimeProvidersFactory?: () => SandboxRuntimeProvider[];
// Sandbox URL provider for browser extensions (optional)
@property({ attribute: false }) sandboxUrlProvider?: () => string;
// Callbacks
@ -108,7 +108,8 @@ export class ArtifactsPanel extends LitElement {
const type = this.getFileType(filename);
if (type === "html") {
element = new HtmlArtifact();
(element as HtmlArtifact).attachments = this.attachmentsProvider?.() || [];
const runtimeProviders = this.runtimeProvidersFactory?.() || [];
(element as HtmlArtifact).runtimeProviders = runtimeProviders;
if (this.sandboxUrlProvider) {
(element as HtmlArtifact).sandboxUrlProvider = this.sandboxUrlProvider;
}
@ -144,7 +145,8 @@ export class ArtifactsPanel extends LitElement {
element.content = content;
element.displayTitle = title;
if (element instanceof HtmlArtifact) {
element.attachments = this.attachmentsProvider?.() || [];
const runtimeProviders = this.runtimeProvidersFactory?.() || [];
element.runtimeProviders = runtimeProviders;
}
}
@ -414,46 +416,11 @@ CRITICAL REMINDER FOR ALL ARTIFACTS:
}
return new Promise((resolve) => {
let resolved = false;
// Listen for the execution-complete message
const messageHandler = (event: MessageEvent) => {
if (event.data?.type === "execution-complete" && event.data?.artifactId === filename) {
if (!resolved) {
resolved = true;
window.removeEventListener("message", messageHandler);
// Get the logs from the element
const logs = element.getLogs();
if (logs.includes("[error]")) {
resolve(`\n\nExecution completed with errors:\n${logs}`);
} else if (logs !== `No logs for ${filename}`) {
resolve(`\n\nExecution logs:\n${logs}`);
} else {
resolve("");
}
}
}
};
window.addEventListener("message", messageHandler);
// Fallback timeout in case the message never arrives
// Fallback timeout - just get logs after execution should complete
setTimeout(() => {
if (!resolved) {
resolved = true;
window.removeEventListener("message", messageHandler);
// Get whatever logs we have so far
const logs = element.getLogs();
if (logs.includes("[error]")) {
resolve(`\n\nExecution timed out with errors:\n${logs}`);
} else if (logs !== `No logs for ${filename}`) {
resolve(`\n\nExecution timed out. Partial logs:\n${logs}`);
} else {
resolve("");
}
}
// Get whatever logs we have
const logs = element.getLogs();
resolve(logs);
}, 1500);
});
}
@ -568,7 +535,7 @@ CRITICAL REMINDER FOR ALL ARTIFACTS:
this.showArtifact(params.filename);
// For HTML files, wait for execution
let result = `Rewrote file ${params.filename}`;
let result = "";
if (this.getFileType(params.filename) === "html" && !options.skipWait) {
const logs = await this.waitForHtmlExecution(params.filename);
result += logs;

23
packages/web-ui/src/tools/javascript-repl.ts
View file

@ -4,15 +4,15 @@ import { type Static, Type } from "@sinclair/typebox";
import { createRef, ref } from "lit/directives/ref.js";
import { Code } from "lucide";
import { type SandboxFile, SandboxIframe, type SandboxResult } from "../components/SandboxedIframe.js";
import type { SandboxRuntimeProvider } from "../components/sandbox/SandboxRuntimeProvider.js";
import type { Attachment } from "../utils/attachment-utils.js";
import { registerToolRenderer, renderCollapsibleHeader, renderHeader } from "./renderer-registry.js";
import type { ToolRenderer } from "./types.js";
// Execute JavaScript code with attachments using SandboxedIframe
export async function executeJavaScript(
code: string,
attachments: Attachment[] = [],
runtimeProviders: SandboxRuntimeProvider[],
signal?: AbortSignal,
sandboxUrlProvider?: () => string,
): Promise<{ output: string; files?: SandboxFile[] }> {
@ -34,8 +34,11 @@ export async function executeJavaScript(
document.body.appendChild(sandbox);
try {
const sandboxId = `repl-${Date.now()}`;
const result: SandboxResult = await sandbox.execute(sandboxId, code, attachments, signal);
const sandboxId = `repl-${Date.now()}-${Math.random().toString(36).substring(7)}`;
// Pass providers to execute (router handles all message routing)
// No additional consumers needed - execute() has its own internal consumer
const result: SandboxResult = await sandbox.execute(sandboxId, code, runtimeProviders, [], signal);
// Remove the sandbox iframe after execution
sandbox.remove();
@ -114,13 +117,13 @@ interface JavaScriptReplResult {
}
export function createJavaScriptReplTool(): AgentTool<typeof javascriptReplSchema, JavaScriptReplToolResult> & {
attachmentsProvider?: () => Attachment[];
runtimeProvidersFactory?: () => SandboxRuntimeProvider[];
sandboxUrlProvider?: () => string;
} {
return {
label: "JavaScript REPL",
name: "javascript_repl",
attachmentsProvider: () => [], // default to empty array
runtimeProvidersFactory: () => [], // default to empty array
sandboxUrlProvider: undefined, // optional, for browser extensions
description: `Execute JavaScript code in a sandboxed browser environment with full modern browser capabilities.
@ -196,8 +199,12 @@ Global variables:
- All standard browser globals (window, document, fetch, etc.)`,
parameters: javascriptReplSchema,
execute: async function (_toolCallId: string, args: Static<typeof javascriptReplSchema>, signal?: AbortSignal) {
const attachments = this.attachmentsProvider?.() || [];
const result = await executeJavaScript(args.code, attachments, signal, this.sandboxUrlProvider);
const result = await executeJavaScript(
args.code,
this.runtimeProvidersFactory?.() ?? [],
signal,
this.sandboxUrlProvider,
);
// Convert files to JSON-serializable with base64 payloads
const files = (result.files || []).map((f) => {
const toBase64 = (input: string | Uint8Array): { base64: string; size: number } => {