From 0eaa879d4609ae2c42b1423a2b95aea8f719b558 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 8 Oct 2025 22:51:32 +0200 Subject: [PATCH] Fix various sandbox issues. --- packages/web-ui/src/ChatPanel.ts | 49 +- .../web-ui/src/components/SandboxedIframe.ts | 472 ++++++------------ .../sandbox/AttachmentsRuntimeProvider.ts | 99 ++++ .../sandbox/ConsoleRuntimeProvider.ts | 132 +++++ .../sandbox/SandboxMessageRouter.ts | 152 ++++++ .../sandbox/SandboxRuntimeProvider.ts | 30 ++ .../src/tools/artifacts/HtmlArtifact.ts | 66 ++- .../artifacts/artifacts-tool-renderer.ts | 24 +- .../web-ui/src/tools/artifacts/artifacts.ts | 57 +-- packages/web-ui/src/tools/javascript-repl.ts | 23 +- 10 files changed, 673 insertions(+), 431 deletions(-) create mode 100644 packages/web-ui/src/components/sandbox/AttachmentsRuntimeProvider.ts create mode 100644 packages/web-ui/src/components/sandbox/ConsoleRuntimeProvider.ts create mode 100644 packages/web-ui/src/components/sandbox/SandboxMessageRouter.ts create mode 100644 packages/web-ui/src/components/sandbox/SandboxRuntimeProvider.ts diff --git a/packages/web-ui/src/ChatPanel.ts b/packages/web-ui/src/ChatPanel.ts index e8c728ab..9db4eb74 100644 --- a/packages/web-ui/src/ChatPanel.ts +++ b/packages/web-ui/src/ChatPanel.ts @@ -1,12 +1,15 @@ import { Badge, html } from "@mariozechner/mini-lit"; import { LitElement } from "lit"; import { customElement, property, state } from "lit/decorators.js"; -import type { AgentInterface } from "./components/AgentInterface.js"; -import "./components/AgentInterface.js"; import type { Agent } from "./agent/agent.js"; +import "./components/AgentInterface.js"; +import type { AgentInterface } from "./components/AgentInterface.js"; +import { AttachmentsRuntimeProvider } from "./components/sandbox/AttachmentsRuntimeProvider.js"; +import type { SandboxRuntimeProvider } from "./components/sandbox/SandboxRuntimeProvider.js"; import { ArtifactsPanel, ArtifactsToolRenderer } from "./tools/artifacts/index.js"; import { createJavaScriptReplTool } from "./tools/javascript-repl.js"; import { registerToolRenderer } from "./tools/renderer-registry.js"; +import type { Attachment } from "./utils/attachment-utils.js"; import { i18n } from "./utils/i18n.js"; const BREAKPOINT = 800; // px - switch between overlay and side-by-side @@ -20,6 +23,21 @@ export class ChatPanel extends LitElement { @state() private artifactCount = 0; @state() private showArtifactsPanel = false; @state() private windowWidth = 0; + @property({ attribute: false }) runtimeProvidersFactory = () => { + const attachments: Attachment[] = []; + for (const message of this.agent!.state.messages) { + if (message.role === "user") { + message.attachments?.forEach((a) => { + attachments.push(a); + }); + } + } + const providers: SandboxRuntimeProvider[] = []; + if (attachments.length > 0) { + providers.push(new AttachmentsRuntimeProvider(attachments)); + } + return providers; + }; @property({ attribute: false }) sandboxUrlProvider?: () => string; @property({ attribute: false }) onApiKeyRequired?: (provider: string) => Promise; @property({ attribute: false }) onBeforeSend?: () => void | Promise; @@ -81,30 +99,9 @@ export class ChatPanel extends LitElement { // Register the standalone tool renderer (not the panel itself) registerToolRenderer("artifacts", new ArtifactsToolRenderer(this.artifactsPanel)); - // Attachments provider - const getAttachments = () => { - const attachments: any[] = []; - for (const message of this.agent!.state.messages) { - if (message.role === "user") { - const content = Array.isArray(message.content) ? message.content : [message.content]; - for (const block of content) { - if (typeof block !== "string" && block.type === "image") { - attachments.push({ - id: `image-${attachments.length}`, - fileName: "image.png", - mimeType: block.mimeType || "image/png", - size: 0, - content: block.data, - }); - } - } - } - } - return attachments; - }; - - javascriptReplTool.attachmentsProvider = getAttachments; - this.artifactsPanel.attachmentsProvider = getAttachments; + // Runtime providers factory + javascriptReplTool.runtimeProvidersFactory = this.runtimeProvidersFactory; + this.artifactsPanel.runtimeProvidersFactory = this.runtimeProvidersFactory; this.artifactsPanel.onArtifactsChange = () => { const count = this.artifactsPanel?.artifacts?.size ?? 0; diff --git a/packages/web-ui/src/components/SandboxedIframe.ts b/packages/web-ui/src/components/SandboxedIframe.ts index 24990c2d..4f17c267 100644 --- a/packages/web-ui/src/components/SandboxedIframe.ts +++ b/packages/web-ui/src/components/SandboxedIframe.ts @@ -1,6 +1,8 @@ import { LitElement } from "lit"; import { customElement, property } from "lit/decorators.js"; -import type { Attachment } from "../utils/attachment-utils.js"; +import { ConsoleRuntimeProvider } from "./sandbox/ConsoleRuntimeProvider.js"; +import { type MessageConsumer, SANDBOX_MESSAGE_ROUTER } from "./sandbox/SandboxMessageRouter.js"; +import type { SandboxRuntimeProvider } from "./sandbox/SandboxRuntimeProvider.js"; export interface SandboxFile { fileName: string; @@ -42,6 +44,9 @@ export class SandboxIframe extends LitElement { override disconnectedCallback() { super.disconnectedCallback(); + // Note: We don't unregister the sandbox here for loadContent() mode + // because the caller (HtmlArtifact) owns the sandbox lifecycle. + // For execute() mode, the sandbox is unregistered in the cleanup function. this.iframe?.remove(); } @@ -49,65 +54,88 @@ export class SandboxIframe extends LitElement { * Load HTML content into sandbox and keep it displayed (for HTML artifacts) * @param sandboxId Unique ID * @param htmlContent Full HTML content - * @param attachments Attachments available + * @param providers Runtime providers to inject + * @param consumers Message consumers to register (optional) */ - public loadContent(sandboxId: string, htmlContent: string, attachments: Attachment[]): void { - const completeHtml = this.prepareHtmlDocument(sandboxId, htmlContent, attachments); + public loadContent( + sandboxId: string, + htmlContent: string, + providers: SandboxRuntimeProvider[] = [], + consumers: MessageConsumer[] = [], + ): void { + // Unregister previous sandbox if exists + try { + SANDBOX_MESSAGE_ROUTER.unregisterSandbox(sandboxId); + } catch { + // Sandbox might not exist, that's ok + } + + providers = [new ConsoleRuntimeProvider(), ...providers]; + + SANDBOX_MESSAGE_ROUTER.registerSandbox(sandboxId, providers, consumers); + + const completeHtml = this.prepareHtmlDocument(sandboxId, htmlContent, providers); + + // Remove previous iframe if exists + this.iframe?.remove(); if (this.sandboxUrlProvider) { // Browser extension mode: use sandbox.html with postMessage - this.loadViaSandboxUrl(sandboxId, completeHtml, attachments); + this.loadViaSandboxUrl(sandboxId, completeHtml); } else { // Web mode: use srcdoc - this.loadViaSrcdoc(completeHtml); + this.loadViaSrcdoc(sandboxId, completeHtml); } } - private loadViaSandboxUrl(sandboxId: string, completeHtml: string, attachments: Attachment[]): void { - // Wait for sandbox-ready and send content + private loadViaSandboxUrl(sandboxId: string, completeHtml: string): void { + // Create iframe pointing to sandbox URL + this.iframe = document.createElement("iframe"); + this.iframe.sandbox.add("allow-scripts"); + this.iframe.sandbox.add("allow-modals"); + this.iframe.style.width = "100%"; + this.iframe.style.height = "100%"; + this.iframe.style.border = "none"; + this.iframe.src = this.sandboxUrlProvider!(); + + // Update router with iframe reference BEFORE appending to DOM + SANDBOX_MESSAGE_ROUTER.setSandboxIframe(sandboxId, this.iframe); + + // Listen for sandbox-ready message directly const readyHandler = (e: MessageEvent) => { if (e.data.type === "sandbox-ready" && e.source === this.iframe?.contentWindow) { window.removeEventListener("message", readyHandler); + + // Send content to sandbox this.iframe?.contentWindow?.postMessage( { type: "sandbox-load", sandboxId, code: completeHtml, - attachments, }, "*", ); } }; + window.addEventListener("message", readyHandler); - // Always recreate iframe to ensure fresh sandbox and sandbox-ready message - this.iframe?.remove(); - this.iframe = document.createElement("iframe"); - this.iframe.sandbox.add("allow-scripts"); - this.iframe.sandbox.add("allow-modals"); - this.iframe.style.width = "100%"; - this.iframe.style.height = "100%"; - this.iframe.style.border = "none"; - - this.iframe.src = this.sandboxUrlProvider!(); - this.appendChild(this.iframe); } - private loadViaSrcdoc(completeHtml: string): void { - // Always recreate iframe to ensure fresh sandbox - this.iframe?.remove(); + private loadViaSrcdoc(sandboxId: string, completeHtml: string): void { + // Create iframe with srcdoc this.iframe = document.createElement("iframe"); this.iframe.sandbox.add("allow-scripts"); this.iframe.sandbox.add("allow-modals"); this.iframe.style.width = "100%"; this.iframe.style.height = "100%"; this.iframe.style.border = "none"; - - // Set content directly via srcdoc (no CSP restrictions in web apps) this.iframe.srcdoc = completeHtml; + // Update router with iframe reference BEFORE appending to DOM + SANDBOX_MESSAGE_ROUTER.setSandboxIframe(sandboxId, this.iframe); + this.appendChild(this.iframe); } @@ -115,143 +143,141 @@ export class SandboxIframe extends LitElement { * Execute code in sandbox * @param sandboxId Unique ID for this execution * @param code User code (plain JS for REPL, or full HTML for artifacts) - * @param attachments Attachments available to the code + * @param providers Runtime providers to inject + * @param consumers Additional message consumers (optional, execute has its own internal consumer) * @param signal Abort signal * @returns Promise resolving to execution result */ public async execute( sandboxId: string, code: string, - attachments: Attachment[], + providers: SandboxRuntimeProvider[] = [], + consumers: MessageConsumer[] = [], signal?: AbortSignal, ): Promise { if (signal?.aborted) { throw new Error("Execution aborted"); } - // Prepare the complete HTML document with runtime + user code - const completeHtml = this.prepareHtmlDocument(sandboxId, code, attachments); + providers = [new ConsoleRuntimeProvider(), ...providers]; + SANDBOX_MESSAGE_ROUTER.registerSandbox(sandboxId, providers, consumers); + + const logs: Array<{ type: string; text: string }> = []; + const files: SandboxFile[] = []; + let completed = false; - // Wait for execution to complete return new Promise((resolve, reject) => { - const logs: Array<{ type: string; text: string }> = []; - const files: SandboxFile[] = []; - let completed = false; - - const messageHandler = (e: MessageEvent) => { - // Ignore messages not for this sandbox - if (e.data.sandboxId !== sandboxId) return; - - if (e.data.type === "console") { - logs.push({ - type: e.data.method === "error" ? "error" : "log", - text: e.data.text, - }); - } else if (e.data.type === "file-returned") { - files.push({ - fileName: e.data.fileName, - content: e.data.content, - mimeType: e.data.mimeType, - }); - } else if (e.data.type === "execution-complete") { - completed = true; - cleanup(); - resolve({ - success: true, - console: logs, - files: files, - }); - } else if (e.data.type === "execution-error") { - completed = true; - cleanup(); - resolve({ - success: false, - console: logs, - error: e.data.error, - files, - }); - } + // 4. Create execution consumer for lifecycle messages + const executionConsumer: MessageConsumer = { + handleMessage(message: any): boolean { + if (message.type === "console") { + logs.push({ + type: message.method === "error" ? "error" : "log", + text: message.text, + }); + return true; + } else if (message.type === "file-returned") { + files.push({ + fileName: message.fileName, + content: message.content, + mimeType: message.mimeType, + }); + return true; + } else if (message.type === "execution-complete") { + completed = true; + cleanup(); + resolve({ success: true, console: logs, files }); + return true; + } else if (message.type === "execution-error") { + completed = true; + cleanup(); + resolve({ success: false, console: logs, error: message.error, files }); + return true; + } + return false; + }, }; + SANDBOX_MESSAGE_ROUTER.addConsumer(sandboxId, executionConsumer); + + const cleanup = () => { + SANDBOX_MESSAGE_ROUTER.unregisterSandbox(sandboxId); + signal?.removeEventListener("abort", abortHandler); + clearTimeout(timeoutId); + this.iframe?.remove(); + this.iframe = undefined; + }; + + // Abort handler const abortHandler = () => { if (!completed) { + completed = true; cleanup(); reject(new Error("Execution aborted")); } }; - let readyHandler: ((e: MessageEvent) => void) | undefined; + if (signal) { + signal.addEventListener("abort", abortHandler); + } - const cleanup = () => { - window.removeEventListener("message", messageHandler); - signal?.removeEventListener("abort", abortHandler); - if (readyHandler) { - window.removeEventListener("message", readyHandler); - } - clearTimeout(timeoutId); - }; - - // Set up listeners BEFORE creating iframe - window.addEventListener("message", messageHandler); - signal?.addEventListener("abort", abortHandler); - - // Timeout after 30 seconds + // Timeout handler (30 seconds) const timeoutId = setTimeout(() => { if (!completed) { + completed = true; cleanup(); resolve({ success: false, - error: { message: "Execution timeout (30s)", stack: "" }, console: logs, + error: { message: "Execution timeout (30s)", stack: "" }, files, }); } }, 30000); + // 4. Prepare HTML and create iframe + const completeHtml = this.prepareHtmlDocument(sandboxId, code, providers); + if (this.sandboxUrlProvider) { - // Browser extension mode: wait for sandbox-ready and send content - readyHandler = (e: MessageEvent) => { + // Browser extension mode: wait for sandbox-ready + this.iframe = document.createElement("iframe"); + this.iframe.sandbox.add("allow-scripts", "allow-modals"); + this.iframe.style.cssText = "width: 100%; height: 100%; border: none;"; + this.iframe.src = this.sandboxUrlProvider(); + + // Update router with iframe reference BEFORE appending to DOM + SANDBOX_MESSAGE_ROUTER.setSandboxIframe(sandboxId, this.iframe); + + // Listen for sandbox-ready message directly + const readyHandler = (e: MessageEvent) => { if (e.data.type === "sandbox-ready" && e.source === this.iframe?.contentWindow) { - window.removeEventListener("message", readyHandler!); - // Send the complete HTML + window.removeEventListener("message", readyHandler); + + // Send content to sandbox this.iframe?.contentWindow?.postMessage( { type: "sandbox-load", sandboxId, code: completeHtml, - attachments, }, "*", ); } }; + window.addEventListener("message", readyHandler); - // Create iframe AFTER all listeners are set up - this.iframe?.remove(); - this.iframe = document.createElement("iframe"); - this.iframe.sandbox.add("allow-scripts"); - this.iframe.sandbox.add("allow-modals"); - this.iframe.style.width = "100%"; - this.iframe.style.height = "100%"; - this.iframe.style.border = "none"; - - this.iframe.src = this.sandboxUrlProvider(); - this.appendChild(this.iframe); } else { // Web mode: use srcdoc - this.iframe?.remove(); this.iframe = document.createElement("iframe"); - this.iframe.sandbox.add("allow-scripts"); - this.iframe.sandbox.add("allow-modals"); - this.iframe.style.width = "100%"; - this.iframe.style.height = "100%"; - this.iframe.style.border = "none"; - - // Set content via srcdoc BEFORE appending to DOM + this.iframe.sandbox.add("allow-scripts", "allow-modals"); + this.iframe.style.cssText = "width: 100%; height: 100%; border: none; display: none;"; this.iframe.srcdoc = completeHtml; + // Update router with iframe reference BEFORE appending to DOM + SANDBOX_MESSAGE_ROUTER.setSandboxIframe(sandboxId, this.iframe); + this.appendChild(this.iframe); } }); @@ -259,10 +285,11 @@ export class SandboxIframe extends LitElement { /** * Prepare complete HTML document with runtime + user code + * PUBLIC so HtmlArtifact can use it for download button */ - private prepareHtmlDocument(sandboxId: string, userCode: string, attachments: Attachment[]): string { + public prepareHtmlDocument(sandboxId: string, userCode: string, providers: SandboxRuntimeProvider[] = []): string { // Runtime script that will be injected - const runtime = this.getRuntimeScript(sandboxId, attachments); + const runtime = this.getRuntimeScript(sandboxId, providers); // Check if user provided full HTML const hasHtmlTag = /]*>/i.test(userCode); @@ -311,215 +338,30 @@ export class SandboxIframe extends LitElement { } /** - * Get the runtime script that captures console, provides helpers, etc. + * Generate runtime script from providers */ - private getRuntimeScript(sandboxId: string, attachments: Attachment[]): string { - // Convert attachments to serializable format - const attachmentsData = attachments.map((a) => ({ - id: a.id, - fileName: a.fileName, - mimeType: a.mimeType, - size: a.size, - content: a.content, - extractedText: a.extractedText, - })); + private getRuntimeScript(sandboxId: string, providers: SandboxRuntimeProvider[] = []): string { + // Collect all data from providers + const allData: Record = {}; + for (const provider of providers) { + Object.assign(allData, provider.getData()); + } - // Runtime function that will run in the sandbox (NO parameters - values injected before function) - const runtimeFunc = () => { - // Helper functions - (window as any).listFiles = () => - (attachments || []).map((a: any) => ({ - id: a.id, - fileName: a.fileName, - mimeType: a.mimeType, - size: a.size, - })); + // Collect all runtime functions - pass sandboxId as string literal + const runtimeFunctions: string[] = []; + for (const provider of providers) { + runtimeFunctions.push(`(${provider.getRuntime().toString()})(${JSON.stringify(sandboxId)});`); + } - (window as any).readTextFile = (attachmentId: string) => { - const a = (attachments || []).find((x: any) => x.id === attachmentId); - if (!a) throw new Error("Attachment not found: " + attachmentId); - if (a.extractedText) return a.extractedText; - try { - return atob(a.content); - } catch { - throw new Error("Failed to decode text content for: " + attachmentId); - } - }; + // Build script + const dataInjection = Object.entries(allData) + .map(([key, value]) => `window.${key} = ${JSON.stringify(value)};`) + .join("\n"); - (window as any).readBinaryFile = (attachmentId: string) => { - const a = (attachments || []).find((x: any) => x.id === attachmentId); - if (!a) throw new Error("Attachment not found: " + attachmentId); - const bin = atob(a.content); - const bytes = new Uint8Array(bin.length); - for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); - return bytes; - }; - - (window as any).returnFile = async (fileName: string, content: any, mimeType?: string) => { - let finalContent: any, finalMimeType: string; - - if (content instanceof Blob) { - const arrayBuffer = await content.arrayBuffer(); - finalContent = new Uint8Array(arrayBuffer); - finalMimeType = mimeType || content.type || "application/octet-stream"; - if (!mimeType && !content.type) { - throw new Error( - "returnFile: MIME type is required for Blob content. Please provide a mimeType parameter (e.g., 'image/png').", - ); - } - } else if (content instanceof Uint8Array) { - finalContent = content; - if (!mimeType) { - throw new Error( - "returnFile: MIME type is required for Uint8Array content. Please provide a mimeType parameter (e.g., 'image/png').", - ); - } - finalMimeType = mimeType; - } else if (typeof content === "string") { - finalContent = content; - finalMimeType = mimeType || "text/plain"; - } else { - finalContent = JSON.stringify(content, null, 2); - finalMimeType = mimeType || "application/json"; - } - - window.parent.postMessage( - { - type: "file-returned", - sandboxId, - fileName, - content: finalContent, - mimeType: finalMimeType, - }, - "*", - ); - }; - - // Console capture - const originalConsole = { - log: console.log, - error: console.error, - warn: console.warn, - info: console.info, - }; - - ["log", "error", "warn", "info"].forEach((method) => { - (console as any)[method] = (...args: any[]) => { - const text = args - .map((arg) => { - try { - return typeof arg === "object" ? JSON.stringify(arg) : String(arg); - } catch { - return String(arg); - } - }) - .join(" "); - - window.parent.postMessage( - { - type: "console", - sandboxId, - method, - text, - }, - "*", - ); - - (originalConsole as any)[method].apply(console, args); - }; - }); - - // Track errors for HTML artifacts - let lastError: { message: string; stack: string } | null = null; - - // Error handlers - window.addEventListener("error", (e) => { - const text = - (e.error?.stack || e.message || String(e)) + " at line " + (e.lineno || "?") + ":" + (e.colno || "?"); - - // Store the error - lastError = { - message: e.error?.message || e.message || String(e), - stack: e.error?.stack || text, - }; - - window.parent.postMessage( - { - type: "console", - sandboxId, - method: "error", - text, - }, - "*", - ); - }); - - window.addEventListener("unhandledrejection", (e) => { - const text = "Unhandled promise rejection: " + (e.reason?.message || e.reason || "Unknown error"); - - // Store the error - lastError = { - message: e.reason?.message || String(e.reason) || "Unhandled promise rejection", - stack: e.reason?.stack || text, - }; - - window.parent.postMessage( - { - type: "console", - sandboxId, - method: "error", - text, - }, - "*", - ); - }); - - // Expose complete() method for user code to call - let completionSent = false; - (window as any).complete = (error?: { message: string; stack: string }) => { - if (completionSent) return; - completionSent = true; - - // Use provided error or last caught error - const finalError = error || lastError; - - if (finalError) { - window.parent.postMessage( - { - type: "execution-error", - sandboxId, - error: finalError, - }, - "*", - ); - } else { - window.parent.postMessage( - { - type: "execution-complete", - sandboxId, - }, - "*", - ); - } - }; - - // Fallback timeout for HTML artifacts that don't call complete() - if (document.readyState === "complete" || document.readyState === "interactive") { - setTimeout(() => (window as any).complete(), 2000); - } else { - window.addEventListener("load", () => { - setTimeout(() => (window as any).complete(), 2000); - }); - } - }; - - // Prepend the const declarations, then the function - return ( - `` - ); + return ``; } } diff --git a/packages/web-ui/src/components/sandbox/AttachmentsRuntimeProvider.ts b/packages/web-ui/src/components/sandbox/AttachmentsRuntimeProvider.ts new file mode 100644 index 00000000..a6628dca --- /dev/null +++ b/packages/web-ui/src/components/sandbox/AttachmentsRuntimeProvider.ts @@ -0,0 +1,99 @@ +import type { Attachment } from "../../utils/attachment-utils.js"; +import type { SandboxRuntimeProvider } from "./SandboxRuntimeProvider.js"; + +/** + * Attachments Runtime Provider + * + * OPTIONAL provider that provides file access APIs to sandboxed code. + * Only needed when attachments are present. + */ +export class AttachmentsRuntimeProvider implements SandboxRuntimeProvider { + constructor(private attachments: Attachment[]) {} + + getData(): Record { + const attachmentsData = this.attachments.map((a) => ({ + id: a.id, + fileName: a.fileName, + mimeType: a.mimeType, + size: a.size, + content: a.content, + extractedText: a.extractedText, + })); + + return { attachments: attachmentsData }; + } + + getRuntime(): (sandboxId: string) => void { + // This function will be stringified, so no external references! + return (sandboxId: string) => { + // Helper functions for attachments + (window as any).listFiles = () => + ((window as any).attachments || []).map((a: any) => ({ + id: a.id, + fileName: a.fileName, + mimeType: a.mimeType, + size: a.size, + })); + + (window as any).readTextFile = (attachmentId: string) => { + const a = ((window as any).attachments || []).find((x: any) => x.id === attachmentId); + if (!a) throw new Error("Attachment not found: " + attachmentId); + if (a.extractedText) return a.extractedText; + try { + return atob(a.content); + } catch { + throw new Error("Failed to decode text content for: " + attachmentId); + } + }; + + (window as any).readBinaryFile = (attachmentId: string) => { + const a = ((window as any).attachments || []).find((x: any) => x.id === attachmentId); + if (!a) throw new Error("Attachment not found: " + attachmentId); + const bin = atob(a.content); + const bytes = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); + return bytes; + }; + + (window as any).returnFile = async (fileName: string, content: any, mimeType?: string) => { + let finalContent: any, finalMimeType: string; + + if (content instanceof Blob) { + const arrayBuffer = await content.arrayBuffer(); + finalContent = new Uint8Array(arrayBuffer); + finalMimeType = mimeType || content.type || "application/octet-stream"; + if (!mimeType && !content.type) { + throw new Error( + "returnFile: MIME type is required for Blob content. Please provide a mimeType parameter (e.g., 'image/png').", + ); + } + } else if (content instanceof Uint8Array) { + finalContent = content; + if (!mimeType) { + throw new Error( + "returnFile: MIME type is required for Uint8Array content. Please provide a mimeType parameter (e.g., 'image/png').", + ); + } + finalMimeType = mimeType; + } else if (typeof content === "string") { + finalContent = content; + finalMimeType = mimeType || "text/plain"; + } else { + finalContent = JSON.stringify(content, null, 2); + finalMimeType = mimeType || "application/json"; + } + + window.parent.postMessage( + { + type: "file-returned", + sandboxId, + fileName, + content: finalContent, + mimeType: finalMimeType, + }, + "*", + ); + }; + }; + } +} diff --git a/packages/web-ui/src/components/sandbox/ConsoleRuntimeProvider.ts b/packages/web-ui/src/components/sandbox/ConsoleRuntimeProvider.ts new file mode 100644 index 00000000..d11e00c8 --- /dev/null +++ b/packages/web-ui/src/components/sandbox/ConsoleRuntimeProvider.ts @@ -0,0 +1,132 @@ +import type { SandboxRuntimeProvider } from "./SandboxRuntimeProvider.js"; + +/** + * Console Runtime Provider + * + * REQUIRED provider that should always be included first. + * Provides console capture, error handling, and execution lifecycle management. + */ +export class ConsoleRuntimeProvider implements SandboxRuntimeProvider { + getData(): Record { + // No data needed + return {}; + } + + getRuntime(): (sandboxId: string) => void { + return (sandboxId: string) => { + // Console capture + const originalConsole = { + log: console.log, + error: console.error, + warn: console.warn, + info: console.info, + }; + + ["log", "error", "warn", "info"].forEach((method) => { + (console as any)[method] = (...args: any[]) => { + const text = args + .map((arg) => { + try { + return typeof arg === "object" ? JSON.stringify(arg) : String(arg); + } catch { + return String(arg); + } + }) + .join(" "); + + window.parent.postMessage( + { + type: "console", + sandboxId, + method, + text, + }, + "*", + ); + + (originalConsole as any)[method].apply(console, args); + }; + }); + + // Track errors for HTML artifacts + let lastError: { message: string; stack: string } | null = null; + + // Error handlers + window.addEventListener("error", (e) => { + const text = + (e.error?.stack || e.message || String(e)) + " at line " + (e.lineno || "?") + ":" + (e.colno || "?"); + + lastError = { + message: e.error?.message || e.message || String(e), + stack: e.error?.stack || text, + }; + + window.parent.postMessage( + { + type: "console", + sandboxId, + method: "error", + text, + }, + "*", + ); + }); + + window.addEventListener("unhandledrejection", (e) => { + const text = "Unhandled promise rejection: " + (e.reason?.message || e.reason || "Unknown error"); + + lastError = { + message: e.reason?.message || String(e.reason) || "Unhandled promise rejection", + stack: e.reason?.stack || text, + }; + + window.parent.postMessage( + { + type: "console", + sandboxId, + method: "error", + text, + }, + "*", + ); + }); + + // Expose complete() method for user code to call + let completionSent = false; + (window as any).complete = (error?: { message: string; stack: string }) => { + if (completionSent) return; + completionSent = true; + + const finalError = error || lastError; + + if (finalError) { + window.parent.postMessage( + { + type: "execution-error", + sandboxId, + error: finalError, + }, + "*", + ); + } else { + window.parent.postMessage( + { + type: "execution-complete", + sandboxId, + }, + "*", + ); + } + }; + + // Fallback timeout for HTML artifacts that don't call complete() + if (document.readyState === "complete" || document.readyState === "interactive") { + setTimeout(() => (window as any).complete(), 2000); + } else { + window.addEventListener("load", () => { + setTimeout(() => (window as any).complete(), 2000); + }); + } + }; + } +} diff --git a/packages/web-ui/src/components/sandbox/SandboxMessageRouter.ts b/packages/web-ui/src/components/sandbox/SandboxMessageRouter.ts new file mode 100644 index 00000000..6c3c4450 --- /dev/null +++ b/packages/web-ui/src/components/sandbox/SandboxMessageRouter.ts @@ -0,0 +1,152 @@ +import type { SandboxRuntimeProvider } from "./SandboxRuntimeProvider.js"; + +/** + * Message consumer interface - components that want to receive messages from sandboxes + */ +export interface MessageConsumer { + /** + * Handle a message from a sandbox. + * @returns true if message was consumed (stops propagation), false otherwise + */ + handleMessage(message: any): boolean; +} + +/** + * Sandbox context - tracks active sandboxes and their consumers + */ +interface SandboxContext { + sandboxId: string; + iframe: HTMLIFrameElement | null; // null until setSandboxIframe() + providers: SandboxRuntimeProvider[]; + consumers: Set; +} + +/** + * Centralized message router for all sandbox communication. + * + * This singleton replaces all individual window.addEventListener("message") calls + * with a single global listener that routes messages to the appropriate handlers. + * + * Benefits: + * - Single global listener instead of multiple independent listeners + * - Automatic cleanup when sandboxes are destroyed + * - Support for bidirectional communication (providers) and broadcasting (consumers) + * - Clear lifecycle management + */ +export class SandboxMessageRouter { + private sandboxes = new Map(); + private messageListener: ((e: MessageEvent) => void) | null = null; + + /** + * Register a new sandbox with its runtime providers. + * Call this BEFORE creating the iframe. + */ + registerSandbox(sandboxId: string, providers: SandboxRuntimeProvider[], consumers: MessageConsumer[]): void { + this.sandboxes.set(sandboxId, { + sandboxId, + iframe: null, // Will be set via setSandboxIframe() + providers, + consumers: new Set(consumers), + }); + + // Setup global listener if not already done + this.setupListener(); + console.log("Registered sandbox:", sandboxId); + } + + /** + * Update the iframe reference for a sandbox. + * Call this AFTER creating the iframe. + * This is needed so providers can send responses back to the sandbox. + */ + setSandboxIframe(sandboxId: string, iframe: HTMLIFrameElement): void { + const context = this.sandboxes.get(sandboxId); + if (context) { + context.iframe = iframe; + } + console.log("Set iframe for sandbox:", sandboxId); + } + + /** + * Unregister a sandbox and remove all its consumers. + * Call this when the sandbox is destroyed. + */ + unregisterSandbox(sandboxId: string): void { + this.sandboxes.delete(sandboxId); + + // If no more sandboxes, remove global listener + if (this.sandboxes.size === 0 && this.messageListener) { + window.removeEventListener("message", this.messageListener); + this.messageListener = null; + } + console.log("Unregistered sandbox:", sandboxId); + } + + /** + * Add a message consumer for a sandbox. + * Consumers receive broadcast messages (console, execution-complete, etc.) + */ + addConsumer(sandboxId: string, consumer: MessageConsumer): void { + const context = this.sandboxes.get(sandboxId); + if (context) { + context.consumers.add(consumer); + } + console.log("Added consumer for sandbox:", sandboxId); + } + + /** + * Remove a message consumer from a sandbox. + */ + removeConsumer(sandboxId: string, consumer: MessageConsumer): void { + const context = this.sandboxes.get(sandboxId); + if (context) { + context.consumers.delete(consumer); + } + console.log("Removed consumer for sandbox:", sandboxId); + } + + /** + * Setup the global message listener (called automatically) + */ + private setupListener(): void { + if (this.messageListener) return; + + this.messageListener = (e: MessageEvent) => { + const { sandboxId } = e.data; + if (!sandboxId) return; + + console.log("Router received message for sandbox:", sandboxId, e.data); + + const context = this.sandboxes.get(sandboxId); + if (!context) return; + + // Create respond() function for bidirectional communication + const respond = (response: any) => { + if (!response.sandboxId) response.sandboxId = sandboxId; + context.iframe?.contentWindow?.postMessage(response, "*"); + }; + + // 1. Try provider handlers first (for bidirectional comm like memory) + for (const provider of context.providers) { + if (provider.handleMessage) { + const handled = provider.handleMessage(e.data, respond); + if (handled) return; // Stop if handled + } + } + + // 2. Broadcast to consumers (for one-way messages like console) + for (const consumer of context.consumers) { + const consumed = consumer.handleMessage(e.data); + if (consumed) break; // Stop if consumed + } + }; + + window.addEventListener("message", this.messageListener); + } +} + +/** + * Global singleton instance. + * Import this from wherever you need to interact with the message router. + */ +export const SANDBOX_MESSAGE_ROUTER = new SandboxMessageRouter(); diff --git a/packages/web-ui/src/components/sandbox/SandboxRuntimeProvider.ts b/packages/web-ui/src/components/sandbox/SandboxRuntimeProvider.ts new file mode 100644 index 00000000..e1a4acb1 --- /dev/null +++ b/packages/web-ui/src/components/sandbox/SandboxRuntimeProvider.ts @@ -0,0 +1,30 @@ +/** + * Interface for providing runtime capabilities to sandboxed iframes. + * Each provider injects data and runtime functions into the sandbox context. + */ +export interface SandboxRuntimeProvider { + /** + * Returns data to inject into window scope. + * Keys become window properties (e.g., { attachments: [...] } -> window.attachments) + */ + getData(): Record; + + /** + * Returns a runtime function that will be stringified and executed in the sandbox. + * The function receives sandboxId and has access to data from getData() via window. + * + * IMPORTANT: This function will be converted to string via .toString() and injected + * into the sandbox, so it cannot reference external variables or imports. + */ + getRuntime(): (sandboxId: string) => void; + + /** + * Optional message handler for bidirectional communication. + * Return true if the message was handled, false to let other handlers try. + * + * @param message - The message from the sandbox + * @param respond - Function to send a response back to the sandbox + * @returns true if message was handled, false otherwise + */ + handleMessage?(message: any, respond: (response: any) => void): boolean; +} diff --git a/packages/web-ui/src/tools/artifacts/HtmlArtifact.ts b/packages/web-ui/src/tools/artifacts/HtmlArtifact.ts index ed670607..cd8cb84a 100644 --- a/packages/web-ui/src/tools/artifacts/HtmlArtifact.ts +++ b/packages/web-ui/src/tools/artifacts/HtmlArtifact.ts @@ -5,7 +5,8 @@ import { customElement, property, state } from "lit/decorators.js"; import { createRef, type Ref, ref } from "lit/directives/ref.js"; import { unsafeHTML } from "lit/directives/unsafe-html.js"; import type { SandboxIframe } from "../../components/SandboxedIframe.js"; -import type { Attachment } from "../../utils/attachment-utils.js"; +import { type MessageConsumer, SANDBOX_MESSAGE_ROUTER } from "../../components/sandbox/SandboxMessageRouter.js"; +import type { SandboxRuntimeProvider } from "../../components/sandbox/SandboxRuntimeProvider.js"; import { i18n } from "../../utils/i18n.js"; import "../../components/SandboxedIframe.js"; import { ArtifactElement } from "./ArtifactElement.js"; @@ -16,7 +17,7 @@ import "./Console.js"; export class HtmlArtifact extends ArtifactElement { @property() override filename = ""; @property({ attribute: false }) override displayTitle = ""; - @property({ attribute: false }) attachments: Attachment[] = []; + @property({ attribute: false }) runtimeProviders: SandboxRuntimeProvider[] = []; @property({ attribute: false }) sandboxUrlProvider?: () => string; private _content = ""; @@ -26,9 +27,6 @@ export class HtmlArtifact extends ArtifactElement { private sandboxIframeRef: Ref = createRef(); private consoleRef: Ref = createRef(); - // Store message handler so we can remove it - private messageHandler?: (e: MessageEvent) => void; - @state() private viewMode: "preview" | "code" = "preview"; private setViewMode(mode: "preview" | "code") { @@ -47,11 +45,17 @@ export class HtmlArtifact extends ArtifactElement { copyButton.title = i18n("Copy HTML"); copyButton.showText = false; + // Generate standalone HTML with all runtime code injected for download + const sandbox = this.sandboxIframeRef.value; + const sandboxId = `artifact-${this.filename}`; + const downloadContent = + sandbox?.prepareHtmlDocument(sandboxId, this._content, this.runtimeProviders || []) || this._content; + return html`
${toggle} ${copyButton} - ${DownloadButton({ content: this._content, filename: this.filename, mimeType: "text/html", title: i18n("Download HTML") })} + ${DownloadButton({ content: downloadContent, filename: this.filename, mimeType: "text/html", title: i18n("Download HTML") })}
`; } @@ -79,33 +83,29 @@ export class HtmlArtifact extends ArtifactElement { sandbox.sandboxUrlProvider = this.sandboxUrlProvider; } - // Remove previous message handler if it exists - if (this.messageHandler) { - window.removeEventListener("message", this.messageHandler); - } - const sandboxId = `artifact-${this.filename}`; - // Set up message listener to collect logs - this.messageHandler = (e: MessageEvent) => { - if (e.data.sandboxId !== sandboxId) return; - - if (e.data.type === "console") { - // Create new array reference for Lit reactivity - this.logs = [ - ...this.logs, - { - type: e.data.method === "error" ? "error" : "log", - text: e.data.text, - }, - ]; - this.requestUpdate(); // Re-render to show console - } + // Create consumer for console messages + const consumer: MessageConsumer = { + handleMessage: (message: any): boolean => { + if (message.type === "console") { + // Create new array reference for Lit reactivity + this.logs = [ + ...this.logs, + { + type: message.method === "error" ? "error" : "log", + text: message.text, + }, + ]; + this.requestUpdate(); // Re-render to show console + return true; + } + return false; + }, }; - window.addEventListener("message", this.messageHandler); - // Load content (iframe persists, doesn't get removed) - sandbox.loadContent(sandboxId, html, this.attachments); + // Load content - this handles sandbox registration, consumer registration, and iframe creation + sandbox.loadContent(sandboxId, html, this.runtimeProviders, [consumer]); } override get content(): string { @@ -114,11 +114,9 @@ export class HtmlArtifact extends ArtifactElement { override disconnectedCallback() { super.disconnectedCallback(); - // Clean up message handler when element is removed from DOM - if (this.messageHandler) { - window.removeEventListener("message", this.messageHandler); - this.messageHandler = undefined; - } + // Unregister sandbox when element is removed from DOM + const sandboxId = `artifact-${this.filename}`; + SANDBOX_MESSAGE_ROUTER.unregisterSandbox(sandboxId); } override firstUpdated() { diff --git a/packages/web-ui/src/tools/artifacts/artifacts-tool-renderer.ts b/packages/web-ui/src/tools/artifacts/artifacts-tool-renderer.ts index bf556d26..69251d09 100644 --- a/packages/web-ui/src/tools/artifacts/artifacts-tool-renderer.ts +++ b/packages/web-ui/src/tools/artifacts/artifacts-tool-renderer.ts @@ -93,7 +93,11 @@ export class ArtifactsToolRenderer implements ToolRenderer ${renderCollapsibleHeader(state, FileCode2, renderHeaderWithPill(headerText, filename), contentRef, chevronRef, false)}
- ${content ? html`` : ""} + ${isDiff ? diffContent : content ? html`` : ""} ${ isHtml ? html`` @@ -156,7 +160,7 @@ export class ArtifactsToolRenderer implements ToolRenderer + ${renderCollapsibleHeader(state, FileCode2, renderHeaderWithPill(headerText, filename), contentRef, chevronRef, false)} +
+ ${Diff({ oldText: params.old_str || "", newText: params.new_str || "" })} + ${isHtml && logs ? html`` : ""} +
+
+ `; + } + // For DELETE, just show header return html`
diff --git a/packages/web-ui/src/tools/artifacts/artifacts.ts b/packages/web-ui/src/tools/artifacts/artifacts.ts index d47b7a88..15861c07 100644 --- a/packages/web-ui/src/tools/artifacts/artifacts.ts +++ b/packages/web-ui/src/tools/artifacts/artifacts.ts @@ -6,7 +6,7 @@ import { html, LitElement, type TemplateResult } from "lit"; import { customElement, property, state } from "lit/decorators.js"; import { createRef, type Ref, ref } from "lit/directives/ref.js"; import { X } from "lucide"; -import type { Attachment } from "../../utils/attachment-utils.js"; +import type { SandboxRuntimeProvider } from "../../components/sandbox/SandboxRuntimeProvider.js"; import { i18n } from "../../utils/i18n.js"; import type { ArtifactElement } from "./ArtifactElement.js"; import { HtmlArtifact } from "./HtmlArtifact.js"; @@ -45,8 +45,8 @@ export class ArtifactsPanel extends LitElement { private artifactElements = new Map(); private contentRef: Ref = createRef(); - // External provider for attachments (decouples panel from AgentInterface) - @property({ attribute: false }) attachmentsProvider?: () => Attachment[]; + // External factory for runtime providers (decouples panel from AgentInterface) + @property({ attribute: false }) runtimeProvidersFactory?: () => SandboxRuntimeProvider[]; // Sandbox URL provider for browser extensions (optional) @property({ attribute: false }) sandboxUrlProvider?: () => string; // Callbacks @@ -108,7 +108,8 @@ export class ArtifactsPanel extends LitElement { const type = this.getFileType(filename); if (type === "html") { element = new HtmlArtifact(); - (element as HtmlArtifact).attachments = this.attachmentsProvider?.() || []; + const runtimeProviders = this.runtimeProvidersFactory?.() || []; + (element as HtmlArtifact).runtimeProviders = runtimeProviders; if (this.sandboxUrlProvider) { (element as HtmlArtifact).sandboxUrlProvider = this.sandboxUrlProvider; } @@ -144,7 +145,8 @@ export class ArtifactsPanel extends LitElement { element.content = content; element.displayTitle = title; if (element instanceof HtmlArtifact) { - element.attachments = this.attachmentsProvider?.() || []; + const runtimeProviders = this.runtimeProvidersFactory?.() || []; + element.runtimeProviders = runtimeProviders; } } @@ -414,46 +416,11 @@ CRITICAL REMINDER FOR ALL ARTIFACTS: } return new Promise((resolve) => { - let resolved = false; - - // Listen for the execution-complete message - const messageHandler = (event: MessageEvent) => { - if (event.data?.type === "execution-complete" && event.data?.artifactId === filename) { - if (!resolved) { - resolved = true; - window.removeEventListener("message", messageHandler); - - // Get the logs from the element - const logs = element.getLogs(); - if (logs.includes("[error]")) { - resolve(`\n\nExecution completed with errors:\n${logs}`); - } else if (logs !== `No logs for ${filename}`) { - resolve(`\n\nExecution logs:\n${logs}`); - } else { - resolve(""); - } - } - } - }; - - window.addEventListener("message", messageHandler); - - // Fallback timeout in case the message never arrives + // Fallback timeout - just get logs after execution should complete setTimeout(() => { - if (!resolved) { - resolved = true; - window.removeEventListener("message", messageHandler); - - // Get whatever logs we have so far - const logs = element.getLogs(); - if (logs.includes("[error]")) { - resolve(`\n\nExecution timed out with errors:\n${logs}`); - } else if (logs !== `No logs for ${filename}`) { - resolve(`\n\nExecution timed out. Partial logs:\n${logs}`); - } else { - resolve(""); - } - } + // Get whatever logs we have + const logs = element.getLogs(); + resolve(logs); }, 1500); }); } @@ -568,7 +535,7 @@ CRITICAL REMINDER FOR ALL ARTIFACTS: this.showArtifact(params.filename); // For HTML files, wait for execution - let result = `Rewrote file ${params.filename}`; + let result = ""; if (this.getFileType(params.filename) === "html" && !options.skipWait) { const logs = await this.waitForHtmlExecution(params.filename); result += logs; diff --git a/packages/web-ui/src/tools/javascript-repl.ts b/packages/web-ui/src/tools/javascript-repl.ts index d423b5a6..d85b7102 100644 --- a/packages/web-ui/src/tools/javascript-repl.ts +++ b/packages/web-ui/src/tools/javascript-repl.ts @@ -4,15 +4,15 @@ import { type Static, Type } from "@sinclair/typebox"; import { createRef, ref } from "lit/directives/ref.js"; import { Code } from "lucide"; import { type SandboxFile, SandboxIframe, type SandboxResult } from "../components/SandboxedIframe.js"; +import type { SandboxRuntimeProvider } from "../components/sandbox/SandboxRuntimeProvider.js"; import type { Attachment } from "../utils/attachment-utils.js"; - import { registerToolRenderer, renderCollapsibleHeader, renderHeader } from "./renderer-registry.js"; import type { ToolRenderer } from "./types.js"; // Execute JavaScript code with attachments using SandboxedIframe export async function executeJavaScript( code: string, - attachments: Attachment[] = [], + runtimeProviders: SandboxRuntimeProvider[], signal?: AbortSignal, sandboxUrlProvider?: () => string, ): Promise<{ output: string; files?: SandboxFile[] }> { @@ -34,8 +34,11 @@ export async function executeJavaScript( document.body.appendChild(sandbox); try { - const sandboxId = `repl-${Date.now()}`; - const result: SandboxResult = await sandbox.execute(sandboxId, code, attachments, signal); + const sandboxId = `repl-${Date.now()}-${Math.random().toString(36).substring(7)}`; + + // Pass providers to execute (router handles all message routing) + // No additional consumers needed - execute() has its own internal consumer + const result: SandboxResult = await sandbox.execute(sandboxId, code, runtimeProviders, [], signal); // Remove the sandbox iframe after execution sandbox.remove(); @@ -114,13 +117,13 @@ interface JavaScriptReplResult { } export function createJavaScriptReplTool(): AgentTool & { - attachmentsProvider?: () => Attachment[]; + runtimeProvidersFactory?: () => SandboxRuntimeProvider[]; sandboxUrlProvider?: () => string; } { return { label: "JavaScript REPL", name: "javascript_repl", - attachmentsProvider: () => [], // default to empty array + runtimeProvidersFactory: () => [], // default to empty array sandboxUrlProvider: undefined, // optional, for browser extensions description: `Execute JavaScript code in a sandboxed browser environment with full modern browser capabilities. @@ -196,8 +199,12 @@ Global variables: - All standard browser globals (window, document, fetch, etc.)`, parameters: javascriptReplSchema, execute: async function (_toolCallId: string, args: Static, signal?: AbortSignal) { - const attachments = this.attachmentsProvider?.() || []; - const result = await executeJavaScript(args.code, attachments, signal, this.sandboxUrlProvider); + const result = await executeJavaScript( + args.code, + this.runtimeProvidersFactory?.() ?? [], + signal, + this.sandboxUrlProvider, + ); // Convert files to JSON-serializable with base64 payloads const files = (result.files || []).map((f) => { const toBase64 = (input: string | Uint8Array): { base64: string; size: number } => {