diff --git a/packages/browser-extension/PLAN.md b/packages/browser-extension/PLAN.md deleted file mode 100644 index 78cb01d4..00000000 --- a/packages/browser-extension/PLAN.md +++ /dev/null @@ -1,987 +0,0 @@ -# Porting Plan: genai-workshop-new Chat System to Browser Extension - -## Executive Summary - -Port the complete chat interface, message rendering, streaming, tool execution, and transport system from `genai-workshop-new/src/app` to the browser extension. The goal is to provide a full-featured AI chat interface with: - -1. **Multiple transport options**: Direct API calls OR proxy-based calls -2. **Full message rendering**: Text, thinking blocks, tool calls, attachments, images -3. **Streaming support**: Real-time message streaming with proper batching -4. **Tool execution and rendering**: Extensible tool system with custom renderers -5. **Session management**: State management with persistence -6. **Debug capabilities**: Optional debug view for development - -## Current State Analysis - -### Already Ported to Browser Extension -- ✅ `AttachmentTile.ts` - Display attachment thumbnails -- ✅ `AttachmentOverlay.ts` - Full-screen attachment viewer -- ✅ `MessageEditor.ts` - Input field with attachment support -- ✅ `utils/attachment-utils.ts` - PDF, Office, image processing -- ✅ `utils/i18n.ts` - Internationalization -- ✅ `dialogs/ApiKeysDialog.ts` - API key management -- ✅ `dialogs/ModelSelector.ts` - Model selection dialog -- ✅ `state/KeyStore.ts` - API key storage - -### Available in @mariozechner/mini-lit Package -- ✅ `CodeBlock` - Syntax-highlighted code display -- ✅ `MarkdownBlock` - Markdown rendering -- ✅ `Button`, `Input`, `Select`, `Textarea`, etc. - UI components -- ✅ `ThemeToggle` - Dark/light mode -- ✅ `Dialog` - Base dialog component - -### Needs to Be Ported - -#### Core Chat System -1. **AgentInterface.ts** (325 lines) - - Main chat interface container - - Manages scrolling, auto-scroll behavior - - Coordinates MessageList, StreamingMessageContainer, MessageEditor - - Displays usage stats - - Handles session lifecycle - -2. **MessageList.ts** (78 lines) - - Renders stable (non-streaming) messages - - Uses `repeat()` directive for efficient rendering - - Maps tool results by call ID - - Renders user and assistant messages - -3. **Messages.ts** (286 lines) - - **UserMessage component**: Displays user messages with attachments - - **AssistantMessage component**: Displays assistant messages with text, thinking, tool calls - - **ToolMessage component**: Displays individual tool invocations with debug view - - **ToolMessageDebugView component**: Shows tool call args and results - - **AbortedMessage component**: Shows aborted requests - -4. **StreamingMessageContainer.ts** (95 lines) - - Manages streaming message updates - - Batches updates using `requestAnimationFrame` for performance - - Shows loading indicator during streaming - - Handles immediate updates for clearing - -5. **ConsoleBlock.ts** (62 lines) - - Console-style output display - - Auto-scrolling to bottom - - Copy button functionality - - Used by tool renderers - -#### State Management - -6. **state/agent-session.ts** (282 lines) - - **AgentSession class**: Core state management - - Manages conversation state: messages, model, tools, system prompt, thinking level - - Implements pub/sub pattern for state updates - - Handles message preprocessing (e.g., extracting text from documents) - - Coordinates transport for sending messages - - Collects debug information - - Event types: `state-update`, `error-no-model`, `error-no-api-key` - - Methods: - - `prompt(input, attachments)` - Send user message - - `setModel()`, `setSystemPrompt()`, `setThinkingLevel()`, `setTools()` - - `appendMessage()`, `replaceMessages()`, `clearMessages()` - - `abort()` - Cancel ongoing request - - `subscribe(fn)` - Listen to state changes - -7. **state/session-store.ts** (needs investigation) - - Session persistence to IndexedDB - - Load/save conversation history - - Multiple session management - -#### Transport Layer - -8. **state/transports/types.ts** (17 lines) - - `AgentTransport` interface - - `AgentRunConfig` interface - - Defines contract for transport implementations - -9. **state/transports/proxy-transport.ts** (54 lines) - - **LocalTransport class** (misleadingly named - actually proxy) - - Calls proxy server via `streamSimpleProxy` - - Passes auth token from KeyStore - - Yields events from `agentLoop()` - -10. **NEW: state/transports/direct-transport.ts** (needs creation) - - **DirectTransport class** - - Calls provider APIs directly using API keys from KeyStore - - Uses `@mariozechner/pi-ai`'s `agentLoop()` directly - - No auth token needed - -11. **utils/proxy-client.ts** (285 lines) - - `streamSimpleProxy()` function - - Fetches from `/api/stream` endpoint - - Parses SSE (Server-Sent Events) stream - - Reconstructs partial messages from delta events - - Handles abort signals - - Maps proxy events to `AssistantMessageEvent` - - Detects unauthorized and clears auth token - -12. **NEW: utils/config.ts** (needs creation) - - Transport configuration - - Proxy URL configuration - - Storage key: `transport-mode` ("direct" | "proxy") - - Storage key: `proxy-url` (default: configurable) - -#### Tool System - -13. **tools/index.ts** (40 lines) - - Exports tool functions from `@mariozechner/pi-ai` - - Registers default tool renderers - - Exports `renderToolParams()` and `renderToolResult()` - - Re-exports tool implementations - -14. **tools/types.ts** (needs investigation) - - `ToolRenderer` interface - - Contracts for custom tool renderers - -15. **tools/renderer-registry.ts** (19 lines) - - Global registry: `Map` - - `registerToolRenderer(name, renderer)` function - - `getToolRenderer(name)` function - -16. **tools/renderers/DefaultRenderer.ts** (1162 chars) - - Fallback renderer for unknown tools - - Renders params as JSON - - Renders results as JSON or text - -17. **tools/renderers/CalculateRenderer.ts** (1677 chars) - - Custom renderer for calculate tool - - Shows expression and result - -18. **tools/renderers/GetCurrentTimeRenderer.ts** (1328 chars) - - Custom renderer for time tool - - Shows timezone and formatted time - -19. **tools/renderers/BashRenderer.ts** (1500 chars) - - Custom renderer for bash tool - - Uses ConsoleBlock for output - -20. **tools/javascript-repl.ts** (needs investigation) - - JavaScript REPL tool implementation - - May need adaptation for browser environment - -21. **tools/web-search.ts** (needs investigation) - - Web search tool implementation - - Check if compatible with browser extension - -22. **tools/sleep.ts** (needs investigation) - - Simple sleep/delay tool - -#### Utilities - -23. **utils/format.ts** (needs investigation) - - `formatUsage()` - Format token usage and costs - - Other formatting utilities - -24. **utils/auth-token.ts** (21 lines) - - `getAuthToken()` - Prompt for proxy auth token - - `clearAuthToken()` - Remove from storage - - Uses PromptDialog for input - -25. **dialogs/PromptDialog.ts** (needs investigation) - - Simple text input dialog - - Used for auth token entry - -#### Debug/Development - -26. **DebugView.ts** (needs investigation) - - Debug panel showing request/response details - - ChatML formatting - - SSE event stream - - Timing information (TTFT, total time) - - Optional feature for development - -#### NOT Needed - -- ❌ `demos/` folder - All demo files (ignore) -- ❌ `mini/` folder - All UI components (use @mariozechner/mini-lit instead) -- ❌ `admin/ProxyAdmin.ts` - Proxy server admin (not needed in extension) -- ❌ `CodeBlock.ts` - Available in @mariozechner/mini-lit -- ❌ `MarkdownBlock.ts` - Available in @mariozechner/mini-lit -- ❌ `ScatterPlot.ts` - Demo visualization -- ❌ `tools/artifacts.ts` - Artifact tool (demo feature) -- ❌ `tools/bash-mcp-server.ts` - MCP integration (not feasible in browser) -- ❌ `AttachmentTileList.ts` - Likely superseded by MessageEditor integration - ---- - -## Detailed Porting Tasks - -### Phase 1: Core Message Rendering (Foundation) - -#### Task 1.1: Port ConsoleBlock -**File**: `src/ConsoleBlock.ts` -**Dependencies**: mini-lit icons -**Actions**: -1. Copy `ConsoleBlock.ts` to browser extension -2. Update imports to use `@mariozechner/mini-lit` -3. Replace icon imports with lucide icons: - - `iconCheckLine` → `Check` - - `iconFileCopy2Line` → `Copy` -4. Update i18n strings: - - Add "console", "Copy output", "Copied!" to i18n.ts - -**Verification**: Render `` - -#### Task 1.2: Port Messages.ts (User, Assistant, Tool Components) -**File**: `src/Messages.ts` -**Dependencies**: ConsoleBlock, formatUsage, tool rendering -**Actions**: -1. Copy `Messages.ts` to browser extension -2. Update imports: - - `Button` from `@mariozechner/mini-lit` - - `formatUsage` from utils - - Icons from lucide (ToolsLine, Loader4Line, BugLine) -3. Add new type: `AppMessage` (already have partial in extension) -4. Components to register: - - `user-message` - - `assistant-message` - - `tool-message` - - `tool-message-debug` - - `aborted-message` -5. Update i18n strings: - - "Error:", "Request aborted", "Call", "Result", "(no result)", "Waiting for tool result…", "Call was aborted; no result." -6. Guard all custom element registrations - -**Verification**: -- Render user message with text and attachments -- Render assistant message with text, thinking, tool calls -- Render tool message in pending, success, error states - -#### Task 1.3: Port MessageList -**File**: `src/MessageList.ts` -**Dependencies**: Messages.ts -**Actions**: -1. Copy `MessageList.ts` to browser extension -2. Update imports -3. Uses `repeat()` directive from lit - ensure it's available -4. Register `message-list` element with guard - -**Verification**: Render a list of mixed user/assistant/tool messages - -#### Task 1.4: Port StreamingMessageContainer -**File**: `src/StreamingMessageContainer.ts` -**Dependencies**: Messages.ts -**Actions**: -1. Copy `StreamingMessageContainer.ts` to browser extension -2. Update imports -3. Register `streaming-message-container` element with guard -4. Test batching behavior with rapid updates - -**Verification**: -- Stream messages update smoothly -- Cursor blinks during streaming -- Immediate clear works correctly - ---- - -### Phase 2: Tool System - -#### Task 2.1: Port Tool Types and Registry -**Files**: `src/tools/types.ts`, `src/tools/renderer-registry.ts` -**Actions**: -1. Read `tools/types.ts` to understand `ToolRenderer` interface -2. Copy both files to `src/tools/` -3. Create registry as singleton - -**Verification**: Can register and retrieve renderers - -#### Task 2.2: Port Tool Renderers -**Files**: All `src/tools/renderers/*.ts` -**Actions**: -1. Copy `DefaultRenderer.ts` -2. Copy `CalculateRenderer.ts` -3. Copy `GetCurrentTimeRenderer.ts` -4. Copy `BashRenderer.ts` -5. Update all to use `@mariozechner/mini-lit` and lucide icons -6. Ensure all use ConsoleBlock where needed - -**Verification**: Test each renderer with sample tool calls - -#### Task 2.3: Port Tool Implementations -**Files**: `src/tools/javascript-repl.ts`, `src/tools/web-search.ts`, `src/tools/sleep.ts` -**Actions**: -1. Read each file to assess browser compatibility -2. Port `sleep.ts` (should be trivial) -3. Port `javascript-repl.ts` - may need `new Function()` or eval -4. Port `web-search.ts` - check if it uses fetch or needs adaptation -5. Update `tools/index.ts` to register all renderers and export tools - -**Verification**: Test each tool execution in browser context - ---- - -### Phase 3: Transport Layer - -#### Task 3.1: Port Transport Types -**File**: `src/state/transports/types.ts` -**Actions**: -1. Copy file to `src/state/transports/` -2. Verify types align with pi-ai package - -**Verification**: Types compile correctly - -#### Task 3.2: Port Proxy Client -**File**: `src/utils/proxy-client.ts` -**Dependencies**: auth-token.ts -**Actions**: -1. Copy `proxy-client.ts` to `src/utils/` -2. Update `streamSimpleProxy()` to use configurable proxy URL -3. Read proxy URL from config (default: user-configurable) -4. Update error messages for i18n -5. Add i18n strings: "Proxy error: {status} {statusText}", "Proxy error: {error}", "Auth token is required for proxy transport" - -**Verification**: Can connect to proxy server with auth token - -#### Task 3.3: Port Proxy Transport -**File**: `src/state/transports/proxy-transport.ts` -**Actions**: -1. Copy file to `src/state/transports/` -2. Rename `LocalTransport` to `ProxyTransport` for clarity -3. Update to use `streamSimpleProxy` from proxy-client -4. Integrate with KeyStore for auth token - -**Verification**: Can send message through proxy - -#### Task 3.4: Create Direct Transport -**File**: `src/state/transports/direct-transport.ts` (NEW) -**Actions**: -1. Create new `DirectTransport` class implementing `AgentTransport` -2. Use `agentLoop()` from `@mariozechner/pi-ai` directly -3. Integrate with KeyStore to get API keys per provider -4. Pass API key in options to `agentLoop()` -5. Handle `no-api-key` errors by triggering ApiKeysDialog - -**Example Implementation**: -```typescript -import { agentLoop, type AgentContext, type PromptConfig, type UserMessage } from "@mariozechner/pi-ai"; -import { keyStore } from "../../KeyStore.js"; -import type { AgentRunConfig, AgentTransport } from "./types.js"; - -export class DirectTransport implements AgentTransport { - constructor( - private readonly getMessages: () => Promise, - ) {} - - async *run(userMessage: Message, cfg: AgentRunConfig, signal?: AbortSignal) { - // Get API key from KeyStore - const apiKey = await keyStore.getKey(cfg.model.provider); - if (!apiKey) { - throw new Error("no-api-key"); - } - - const context: AgentContext = { - systemPrompt: cfg.systemPrompt, - messages: await this.getMessages(), - tools: cfg.tools, - }; - - const pc: PromptConfig = { - model: cfg.model, - reasoning: cfg.reasoning, - apiKey, // Direct API key - }; - - // Yield events from agentLoop - for await (const ev of agentLoop(userMessage as unknown as UserMessage, context, pc, signal)) { - yield ev; - } - } -} -``` - -**Verification**: Can send message directly to provider APIs - -#### Task 3.5: Create Transport Configuration -**File**: `src/utils/config.ts` (NEW) -**Actions**: -1. Create transport mode storage: "direct" | "proxy" -2. Create proxy URL storage with default -3. Create getters/setters: - - `getTransportMode()` / `setTransportMode()` - - `getProxyUrl()` / `setProxyUrl()` -4. Store in chrome.storage.local - -**Example**: -```typescript -export type TransportMode = "direct" | "proxy"; - -export async function getTransportMode(): Promise { - const result = await chrome.storage.local.get("transport-mode"); - return (result["transport-mode"] as TransportMode) || "direct"; -} - -export async function setTransportMode(mode: TransportMode): Promise { - await chrome.storage.local.set({ "transport-mode": mode }); -} - -export async function getProxyUrl(): Promise { - const result = await chrome.storage.local.get("proxy-url"); - return result["proxy-url"] || "https://genai.mariozechner.at"; -} - -export async function setProxyUrl(url: string): Promise { - await chrome.storage.local.set({ "proxy-url": url }); -} -``` - -**Verification**: Can read/write transport config - ---- - -### Phase 4: State Management - -#### Task 4.1: Port Utilities -**File**: `src/utils/format.ts` -**Actions**: -1. Read file to identify all formatting functions -2. Copy `formatUsage()` function -3. Copy any other utilities needed by AgentSession -4. Update imports - -**Verification**: Test formatUsage with sample usage data - -#### Task 4.2: Port Auth Token Utils -**File**: `src/utils/auth-token.ts` -**Dependencies**: PromptDialog -**Actions**: -1. Copy file to `src/utils/` -2. Update to use chrome.storage.local -3. Will need PromptDialog (next task) - -**Verification**: Can prompt for and store auth token - -#### Task 4.3: Port/Create PromptDialog -**File**: `src/dialogs/PromptDialog.ts` -**Actions**: -1. Read genai-workshop-new version -2. Adapt to use `@mariozechner/mini-lit` Dialog -3. Create simple input dialog similar to ApiKeysDialog -4. Add `PromptDialog.ask(title, message, defaultValue, isPassword)` static method - -**Verification**: Can prompt for text input - -#### Task 4.4: Port Agent Session -**File**: `src/state/agent-session.ts` -**Dependencies**: Transports, formatUsage, auth-token, DebugView types -**Actions**: -1. Copy `agent-session.ts` to `src/state/` -2. Update imports: - - ProxyTransport from `./transports/proxy-transport.js` - - DirectTransport from `./transports/direct-transport.js` - - Types from pi-ai - - KeyStore, auth-token utils -3. Modify constructor to accept transport mode -4. Create transport based on mode: - - "proxy" → ProxyTransport - - "direct" → DirectTransport -5. Update to use chrome.storage for persistence -6. Add `ThinkingLevel` type -7. Add `AppMessage` type extension for attachments - -**Key modifications**: -```typescript -constructor(opts: AgentSessionOptions & { transportMode?: TransportMode } = { - authTokenProvider: async () => getAuthToken() -}) { - // ... existing state init ... - - const mode = opts.transportMode || await getTransportMode(); - - if (mode === "proxy") { - this.transport = new ProxyTransport( - async () => this.preprocessMessages(), - opts.authTokenProvider - ); - } else { - this.transport = new DirectTransport( - async () => this.preprocessMessages() - ); - } -} -``` - -**Verification**: -- Create session with direct transport, send message -- Create session with proxy transport, send message -- Test abort functionality -- Test state subscription - ---- - -### Phase 5: Main Interface Integration - -#### Task 5.1: Port AgentInterface -**File**: `src/AgentInterface.ts` -**Dependencies**: Everything above -**Actions**: -1. Copy `AgentInterface.ts` to `src/` -2. Update all imports to use ported components -3. Register `agent-interface` custom element with guard -4. Update icons to lucide -5. Add i18n strings: - - "No session available", "No session set", "Hide debug view", "Show debug view" -6. Properties: - - `session` (external AgentSession) - - `enableAttachments` - - `enableModelSelector` - - `enableThinking` - - `showThemeToggle` - - `showDebugToggle` -7. Methods: - - `setInput(text, attachments)` - - `sendMessage(input, attachments)` - -**Verification**: Full chat interface works end-to-end - -#### Task 5.2: Integrate into ChatPanel -**File**: `src/ChatPanel.ts` -**Actions**: -1. Remove current chat implementation -2. Create AgentSession instance -3. Render `` with session -4. Configure: - - `enableAttachments={true}` - - `enableModelSelector={true}` - - `enableThinking={true}` - - `showThemeToggle={false}` (already in header) - - `showDebugToggle={false}` (optional) -5. Remove old MessageEditor integration (now inside AgentInterface) -6. Set system prompt (optional) -7. Set default tools (optional - calculateTool, getCurrentTimeTool) - -**Example**: -```typescript -import { AgentSession } from "./state/agent-session.js"; -import "./AgentInterface.js"; -import { calculateTool, getCurrentTimeTool } from "./tools/index.js"; - -@customElement("chat-panel") -export class ChatPanel extends LitElement { - @state() private session!: AgentSession; - - override async connectedCallback() { - super.connectedCallback(); - - // Create session - this.session = new AgentSession({ - initialState: { - systemPrompt: "You are a helpful AI assistant.", - tools: [calculateTool, getCurrentTimeTool], - }, - authTokenProvider: async () => getAuthToken(), - transportMode: await getTransportMode(), - }); - } - - override render() { - return html` - - `; - } -} -``` - -**Verification**: Full extension works with chat interface - -#### Task 5.3: Create Settings Dialog -**File**: `src/dialogs/SettingsDialog.ts` (NEW) -**Actions**: -1. Create dialog extending DialogBase -2. Sections: - - **Transport Mode**: Radio buttons for "Direct" | "Proxy" - - **Proxy URL**: Input field (only shown if proxy mode) - - **API Keys**: Button to open ApiKeysDialog -3. Save settings to config utils - -**UI Layout**: -``` -┌─────────────────────────────────────┐ -│ Settings [x] │ -├─────────────────────────────────────┤ -│ │ -│ Transport Mode │ -│ ○ Direct (use API keys) │ -│ ● Proxy (use auth token) │ -│ │ -│ Proxy URL │ -│ [https://genai.mariozechner.at ] │ -│ │ -│ [Manage API Keys...] │ -│ │ -│ [Cancel] [Save] │ -└─────────────────────────────────────┘ -``` - -**Verification**: Can toggle transport mode and set proxy URL - -#### Task 5.4: Update Header -**File**: `src/sidepanel.ts` -**Actions**: -1. Change settings button to open SettingsDialog (not ApiKeysDialog directly) -2. SettingsDialog should have button to open ApiKeysDialog - -**Verification**: Settings accessible from header - ---- - -### Phase 6: Optional Features - -#### Task 6.1: Port DebugView (Optional) -**File**: `src/DebugView.ts` -**Actions**: -1. Read full file to understand functionality -2. Copy to `src/` -3. Update imports -4. Format ChatML, SSE events, timing info -5. Add to AgentInterface when `showDebugToggle={true}` - -**Verification**: Debug view shows request/response details - -#### Task 6.2: Port Session Store (Optional) -**File**: `src/utils/session-db.ts` or `src/state/session-store.ts` -**Actions**: -1. Read file to understand IndexedDB usage -2. Create IndexedDB schema for sessions -3. Implement save/load/list/delete operations -4. Add to AgentInterface or ChatPanel -5. Add UI for switching sessions - -**Verification**: Can save and load conversation history - -#### Task 6.3: Add System Prompt Editor (Optional) -**Actions**: -1. Create dialog or expandable textarea -2. Allow editing session.state.systemPrompt -3. Add to settings or main interface - -**Verification**: Can customize system prompt - ---- - -## File Mapping Reference - -### Source → Destination - -| Source File | Destination File | Status | Dependencies | -|------------|------------------|--------|--------------| -| `app/ConsoleBlock.ts` | `src/ConsoleBlock.ts` | ⭕ New | mini-lit, lucide | -| `app/Messages.ts` | `src/Messages.ts` | ⭕ New | ConsoleBlock, formatUsage, tools | -| `app/MessageList.ts` | `src/MessageList.ts` | ⭕ New | Messages.ts | -| `app/StreamingMessageContainer.ts` | `src/StreamingMessageContainer.ts` | ⭕ New | Messages.ts | -| `app/AgentInterface.ts` | `src/AgentInterface.ts` | ⭕ New | All message components | -| `app/state/agent-session.ts` | `src/state/agent-session.ts` | ⭕ New | Transports, formatUsage | -| `app/state/transports/types.ts` | `src/state/transports/types.ts` | ⭕ New | pi-ai | -| `app/state/transports/proxy-transport.ts` | `src/state/transports/proxy-transport.ts` | ⭕ New | proxy-client | -| N/A | `src/state/transports/direct-transport.ts` | ⭕ New | pi-ai, KeyStore | -| `app/utils/proxy-client.ts` | `src/utils/proxy-client.ts` | ⭕ New | auth-token | -| N/A | `src/utils/config.ts` | ⭕ New | chrome.storage | -| `app/utils/format.ts` | `src/utils/format.ts` | ⭕ New | None | -| `app/utils/auth-token.ts` | `src/utils/auth-token.ts` | ⭕ New | PromptDialog | -| `app/tools/types.ts` | `src/tools/types.ts` | ⭕ New | None | -| `app/tools/renderer-registry.ts` | `src/tools/renderer-registry.ts` | ⭕ New | types.ts | -| `app/tools/renderers/DefaultRenderer.ts` | `src/tools/renderers/DefaultRenderer.ts` | ⭕ New | mini-lit | -| `app/tools/renderers/CalculateRenderer.ts` | `src/tools/renderers/CalculateRenderer.ts` | ⭕ New | mini-lit | -| `app/tools/renderers/GetCurrentTimeRenderer.ts` | `src/tools/renderers/GetCurrentTimeRenderer.ts` | ⭕ New | mini-lit | -| `app/tools/renderers/BashRenderer.ts` | `src/tools/renderers/BashRenderer.ts` | ⭕ New | ConsoleBlock | -| `app/tools/javascript-repl.ts` | `src/tools/javascript-repl.ts` | ⭕ New | pi-ai | -| `app/tools/web-search.ts` | `src/tools/web-search.ts` | ⭕ New | pi-ai | -| `app/tools/sleep.ts` | `src/tools/sleep.ts` | ⭕ New | pi-ai | -| `app/tools/index.ts` | `src/tools/index.ts` | ⭕ New | All tools | -| `app/dialogs/PromptDialog.ts` | `src/dialogs/PromptDialog.ts` | ⭕ New | mini-lit | -| N/A | `src/dialogs/SettingsDialog.ts` | ⭕ New | config, ApiKeysDialog | -| `app/DebugView.ts` | `src/DebugView.ts` | ⭕ Optional | highlight.js | -| `app/utils/session-db.ts` | `src/utils/session-db.ts` | ⭕ Optional | IndexedDB | - -### Already in Extension - -| File | Status | Notes | -|------|--------|-------| -| `src/MessageEditor.ts` | ✅ Exists | May need minor updates | -| `src/AttachmentTile.ts` | ✅ Exists | Complete | -| `src/AttachmentOverlay.ts` | ✅ Exists | Complete | -| `src/utils/attachment-utils.ts` | ✅ Exists | Complete | -| `src/dialogs/ModelSelector.ts` | ✅ Exists | May need integration check | -| `src/dialogs/ApiKeysDialog.ts` | ✅ Exists | Complete | -| `src/state/KeyStore.ts` | ✅ Exists | Complete | - ---- - -## Critical Implementation Notes - -### 1. Custom Element Registration Guards - -ALL custom elements must use registration guards to prevent duplicate registration errors: - -```typescript -// Instead of @customElement decorator -export class MyComponent extends LitElement { - // ... component code ... -} - -// At end of file -if (!customElements.get("my-component")) { - customElements.define("my-component", MyComponent); -} -``` - -### 2. Import Path Updates - -When porting, update ALL imports: - -**From genai-workshop-new**: -```typescript -import { Button } from "./mini/Button.js"; -import { iconLoader4Line } from "./mini/icons.js"; -``` - -**To browser extension**: -```typescript -import { Button } from "@mariozechner/mini-lit"; -import { Loader2 } from "lucide"; -import { icon } from "@mariozechner/mini-lit"; -// Use: icon(Loader2, "md") -``` - -### 3. Icon Mapping - -| genai-workshop | lucide | Usage | -|----------------|--------|-------| -| `iconLoader4Line` | `Loader2` | `icon(Loader2, "sm")` | -| `iconToolsLine` | `Wrench` | `icon(Wrench, "md")` | -| `iconBugLine` | `Bug` | `icon(Bug, "sm")` | -| `iconCheckLine` | `Check` | `icon(Check, "sm")` | -| `iconFileCopy2Line` | `Copy` | `icon(Copy, "sm")` | - -### 4. Chrome Extension APIs - -Replace browser APIs where needed: -- `localStorage` → `chrome.storage.local` -- `fetch("/api/...")` → `fetch(proxyUrl + "/api/...")` -- No direct filesystem access - -### 5. Transport Mode Configuration - -Ensure AgentSession can be created with either transport: - -```typescript -// Direct mode (uses API keys from KeyStore) -const session = new AgentSession({ - transportMode: "direct", - authTokenProvider: async () => undefined, // not needed -}); - -// Proxy mode (uses auth token) -const session = new AgentSession({ - transportMode: "proxy", - authTokenProvider: async () => getAuthToken(), -}); -``` - -### 6. i18n Strings to Add - -All UI strings must be in i18n.ts with English and German translations: - -```typescript -// Messages.ts -"Error:", "Request aborted", "Call", "Result", "(no result)", -"Waiting for tool result…", "Call was aborted; no result." - -// ConsoleBlock.ts -"console", "Copy output", "Copied!" - -// AgentInterface.ts -"No session available", "No session set", "Hide debug view", "Show debug view" - -// Transport errors -"Proxy error: {status} {statusText}", "Proxy error: {error}", -"Auth token is required for proxy transport" - -// Settings -"Settings", "Transport Mode", "Direct (use API keys)", -"Proxy (use auth token)", "Proxy URL", "Manage API Keys" -``` - -### 7. TypeScript Configuration - -The extension uses `useDefineForClassFields: false` in tsconfig.base.json. Ensure all ported components are compatible. - -### 8. Build Verification Steps - -After each phase: -1. Run `npm run check` - TypeScript compilation -2. Run `npm run build:chrome` - Chrome extension build -3. Run `npm run build:firefox` - Firefox extension build -4. Load extension in browser and test functionality -5. Check console for errors - -### 9. Proxy URL Configuration - -Default proxy URL should be configurable but default to: -```typescript -const DEFAULT_PROXY_URL = "https://genai.mariozechner.at"; -``` - -Users should be able to change this in settings for self-hosted proxies. - ---- - -## Testing Checklist - -### Phase 1: Message Rendering -- [ ] User messages display with text -- [ ] User messages display with attachments -- [ ] Assistant messages display with text -- [ ] Assistant messages display with thinking blocks -- [ ] Assistant messages display with tool calls -- [ ] Tool messages show pending state with spinner -- [ ] Tool messages show completed state with results -- [ ] Tool messages show error state -- [ ] Tool messages show aborted state -- [ ] Console blocks render output -- [ ] Console blocks auto-scroll -- [ ] Console blocks copy to clipboard - -### Phase 2: Tool System -- [ ] Calculate tool renders expression and result -- [ ] Time tool renders timezone and formatted time -- [ ] Bash tool renders output in console block -- [ ] JavaScript REPL tool executes code -- [ ] Web search tool fetches results -- [ ] Sleep tool delays execution -- [ ] Custom tool renderers can be registered -- [ ] Unknown tools use default renderer -- [ ] Tool debug view shows call args and results - -### Phase 3: Transport Layer -- [ ] Proxy transport connects to server -- [ ] Proxy transport handles auth token -- [ ] Proxy transport streams messages -- [ ] Proxy transport reconstructs partial messages -- [ ] Proxy transport handles abort -- [ ] Proxy transport handles errors -- [ ] Direct transport uses API keys from KeyStore -- [ ] Direct transport calls provider APIs directly -- [ ] Direct transport handles missing API key -- [ ] Direct transport streams messages -- [ ] Direct transport handles abort -- [ ] Transport mode can be switched -- [ ] Proxy URL can be configured - -### Phase 4: State Management -- [ ] AgentSession manages conversation state -- [ ] AgentSession sends messages -- [ ] AgentSession receives streaming updates -- [ ] AgentSession handles tool execution -- [ ] AgentSession handles errors -- [ ] AgentSession can be aborted -- [ ] AgentSession persists state -- [ ] AgentSession supports multiple sessions -- [ ] System prompt can be set -- [ ] Model can be selected -- [ ] Thinking level can be adjusted -- [ ] Tools can be configured -- [ ] Usage stats are tracked - -### Phase 5: Main Interface -- [ ] AgentInterface displays messages -- [ ] AgentInterface handles scrolling -- [ ] AgentInterface enables auto-scroll -- [ ] AgentInterface shows usage stats -- [ ] AgentInterface integrates MessageEditor -- [ ] AgentInterface integrates ModelSelector -- [ ] AgentInterface shows thinking toggle -- [ ] Settings dialog opens -- [ ] Settings dialog saves transport mode -- [ ] Settings dialog saves proxy URL -- [ ] Settings dialog opens API keys dialog -- [ ] Header settings button works - -### Phase 6: Optional Features -- [ ] Debug view shows request details -- [ ] Debug view shows response details -- [ ] Debug view shows timing info -- [ ] Debug view formats ChatML -- [ ] Sessions can be saved -- [ ] Sessions can be loaded -- [ ] Sessions can be listed -- [ ] Sessions can be deleted -- [ ] System prompt can be edited - ---- - -## Dependencies to Install - -```bash -# If not already installed -npm install highlight.js # For DebugView (optional) -``` - ---- - -## Estimated Complexity - -| Phase | Files | LOC (approx) | Complexity | Time Estimate | -|-------|-------|--------------|------------|---------------| -| Phase 1 | 5 | ~800 | Medium | 4-6 hours | -| Phase 2 | 10 | ~400 | Low-Medium | 3-4 hours | -| Phase 3 | 6 | ~600 | High | 6-8 hours | -| Phase 4 | 5 | ~500 | Medium-High | 5-7 hours | -| Phase 5 | 4 | ~400 | Medium | 4-5 hours | -| Phase 6 | 3 | ~400 | Low | 2-3 hours | -| **TOTAL** | **33** | **~3100** | - | **24-33 hours** | - ---- - -## Success Criteria - -The port is complete when: - -1. ✅ User can send messages with text and attachments -2. ✅ Messages stream in real-time with proper rendering -3. ✅ Tool calls execute and display results -4. ✅ Both direct and proxy transports work -5. ✅ Settings can be configured and persisted -6. ✅ Usage stats are tracked and displayed -7. ✅ Extension works in both Chrome and Firefox -8. ✅ All TypeScript types compile without errors -9. ✅ No console errors in normal operation -10. ✅ UI is responsive and performs well - ---- - -## Notes for New Session - -If starting a new session, key context: - -1. **Extension structure**: Browser extension in `packages/browser-extension/` -2. **Source codebase**: `genai-workshop-new/src/app/` -3. **UI framework**: LitElement with `@mariozechner/mini-lit` package -4. **AI package**: `@mariozechner/pi-ai` for LLM interactions -5. **Icons**: Using lucide instead of custom icon set -6. **i18n**: All UI strings must be in i18n.ts (English + German) -7. **Storage**: chrome.storage.local for all persistence -8. **TypeScript**: `useDefineForClassFields: false` required -9. **Custom elements**: Must use registration guards -10. **Build**: `npm run build:chrome` and `npm run build:firefox` - -**Critical files to reference**: -- `packages/browser-extension/tsconfig.json` - TS config -- `packages/browser-extension/src/utils/i18n.ts` - i18n strings -- `packages/browser-extension/src/state/KeyStore.ts` - API key storage -- `packages/browser-extension/src/dialogs/ApiKeysDialog.ts` - API key UI -- `genai-workshop-new/src/app/AgentInterface.ts` - Reference implementation -- `genai-workshop-new/src/app/state/agent-session.ts` - State management reference - -**Key architectural decisions**: -- Single AgentSession per chat -- Transport is pluggable (direct or proxy) -- Tools are registered in a global registry -- Message rendering is separated: stable (MessageList) vs streaming (StreamingMessageContainer) -- All components use light DOM (`createRenderRoot() { return this; }`) diff --git a/packages/browser-extension/README.md b/packages/browser-extension/README.md index e81d5ad8..1211c964 100644 --- a/packages/browser-extension/README.md +++ b/packages/browser-extension/README.md @@ -5,7 +5,7 @@ A cross-browser extension that provides an AI-powered reading assistant in a sid ## Browser Support - **Chrome/Edge** - Uses Side Panel API (Manifest V3) -- **Firefox** - Uses Sidebar Action API (Manifest V3) +- **Firefox** - Uses Sidebar Action API (Manifest V2) - **Opera** - Sidebar support (untested but should work with Firefox manifest) ## Architecture @@ -18,9 +18,10 @@ The extension is a full-featured AI chat interface that runs in your browser's s 2. **Proxy Mode** - Routes requests through a proxy server using an auth token **Browser Adaptation:** -- **Chrome/Edge** - Side Panel API for dedicated panel UI -- **Firefox** - Sidebar Action API for sidebar UI +- **Chrome/Edge** - Side Panel API for dedicated panel UI, Manifest V3 +- **Firefox** - Sidebar Action API for sidebar UI, Manifest V2 - **Page Content Access** - Uses `chrome.scripting.executeScript` to extract page text +- **Cross-browser APIs** - Uses `browser.*` (Firefox) and `chrome.*` (Chrome/Edge) via runtime detection ### Core Architecture Layers @@ -72,6 +73,10 @@ src/ │ ├── AttachmentOverlay.ts # Full-screen attachment viewer │ └── ModeToggle.ts # Toggle between document/text view │ +├── Components (reusable utilities) +│ └── components/ +│ └── SandboxedIframe.ts # Sandboxed HTML renderer with console capture +│ ├── Dialogs (modal interactions) │ ├── dialogs/ │ │ ├── DialogBase.ts # Base class for all dialogs @@ -82,7 +87,7 @@ src/ ├── State Management (business logic) │ ├── state/ │ │ ├── agent-session.ts # Core state manager (pub/sub pattern) -│ │ ├── KeyStore.ts # API key storage (Chrome local storage) +│ │ ├── KeyStore.ts # Cross-browser API key storage │ │ └── transports/ │ │ ├── types.ts # Transport interface definitions │ │ ├── DirectTransport.ts # Direct API calls @@ -93,11 +98,16 @@ src/ │ │ ├── types.ts # ToolRenderer interface │ │ ├── renderer-registry.ts # Global tool renderer registry │ │ ├── index.ts # Tool exports and registration -│ │ └── renderers/ # Custom tool UI renderers -│ │ ├── DefaultRenderer.ts # Fallback for unknown tools -│ │ ├── CalculateRenderer.ts # Calculator tool UI -│ │ ├── GetCurrentTimeRenderer.ts -│ │ └── BashRenderer.ts # Bash command execution UI +│ │ ├── browser-javascript.ts # Execute JS in current tab +│ │ ├── renderers/ # Custom tool UI renderers +│ │ │ ├── DefaultRenderer.ts # Fallback for unknown tools +│ │ │ ├── CalculateRenderer.ts # Calculator tool UI +│ │ │ ├── GetCurrentTimeRenderer.ts +│ │ │ └── BashRenderer.ts # Bash command execution UI +│ │ └── artifacts/ # Artifact tools (HTML, Mermaid, etc.) +│ │ ├── ArtifactElement.ts # Base class for artifacts +│ │ ├── HtmlArtifact.ts # HTML artifact with sandboxed preview +│ │ └── MermaidArtifact.ts # Mermaid diagram rendering │ ├── Utilities (shared helpers) │ └── utils/ @@ -109,6 +119,8 @@ src/ └── Entry Points (browser integration) ├── background.ts # Service worker (opens side panel) ├── sidepanel.html # HTML entry point + ├── sandbox.html # Sandboxed page for artifact HTML + ├── sandbox.js # Sandbox environment setup └── live-reload.ts # Hot reload during development ``` @@ -163,6 +175,7 @@ import type { ToolRenderer } from "../types.js"; export class MyCustomRenderer implements ToolRenderer { renderParams(params: any, isStreaming?: boolean) { // Show tool call parameters (e.g., "Searching for: ") + return html`
${isStreaming ? "Processing..." : `Input: ${params.input}`} @@ -233,9 +246,9 @@ this.session = new AgentSession({ **Message components** control how conversations appear: -- **User messages**: Edit `UserMessage` in `src/Messages.ts` -- **Assistant messages**: Edit `AssistantMessage` in `src/Messages.ts` -- **Tool call cards**: Edit `ToolMessage` in `src/Messages.ts` +- **User messages**: Edit `UserMessage` in [src/Messages.ts](src/Messages.ts) +- **Assistant messages**: Edit `AssistantMessage` in [src/Messages.ts](src/Messages.ts) +- **Tool call cards**: Edit `ToolMessage` in [src/Messages.ts](src/Messages.ts) - **Markdown rendering**: Comes from `@mariozechner/mini-lit` (can't customize easily) - **Code blocks**: Comes from `@mariozechner/mini-lit` (can't customize easily) @@ -266,7 +279,7 @@ Models come from `@mariozechner/pi-ai`. The package supports: **To add a provider:** 1. Ensure `@mariozechner/pi-ai` supports it (check package docs) -2. Add API key configuration in `src/dialogs/ApiKeysDialog.ts`: +2. Add API key configuration in [src/dialogs/ApiKeysDialog.ts](src/dialogs/ApiKeysDialog.ts): - Add provider to `PROVIDERS` array - Add test model to `TEST_MODELS` object 3. Users can then select models via the model selector @@ -280,13 +293,13 @@ Models come from `@mariozechner/pi-ai`. The package supports: **Transport** determines how requests reach AI providers: #### Direct Mode (Default) -- **File**: `src/state/transports/DirectTransport.ts` +- **File**: [src/state/transports/DirectTransport.ts](src/state/transports/DirectTransport.ts) - **How it works**: Gets API keys from `KeyStore` → calls provider APIs directly - **When to use**: Local development, no proxy server - **Configuration**: API keys stored in Chrome local storage #### Proxy Mode -- **File**: `src/state/transports/ProxyTransport.ts` +- **File**: [src/state/transports/ProxyTransport.ts](src/state/transports/ProxyTransport.ts) - **How it works**: Gets auth token → sends request to proxy server → proxy calls providers - **When to use**: Want to hide API keys, centralized auth, usage tracking - **Configuration**: Auth token stored in localStorage, proxy URL hardcoded @@ -321,7 +334,7 @@ this.session = new AgentSession({ ### "I want to change the system prompt" -**System prompts** guide the AI's behavior. Change in `ChatPanel.ts`: +**System prompts** guide the AI's behavior. Change in [src/ChatPanel.ts](src/ChatPanel.ts): ```typescript // src/ChatPanel.ts @@ -344,7 +357,7 @@ const systemPrompt = await chrome.storage.local.get("system-prompt"); ### "I want to add attachment support for a new file type" -**Attachment processing** happens in `src/utils/attachment-utils.ts`: +**Attachment processing** happens in [src/utils/attachment-utils.ts](src/utils/attachment-utils.ts): 1. **Add file type detection** in `loadAttachment()`: ```typescript @@ -363,12 +376,12 @@ const systemPrompt = await chrome.storage.local.get("system-prompt"); } ``` -3. **Update accepted types** in `MessageEditor.ts`: +3. **Update accepted types** in [src/MessageEditor.ts](src/MessageEditor.ts): ```typescript acceptedTypes = "image/*,application/pdf,.myext,..."; ``` -4. **Optional: Add preview support** in `AttachmentOverlay.ts` +4. **Optional: Add preview support** in [src/AttachmentOverlay.ts](src/AttachmentOverlay.ts) **Supported formats:** - Images: All image/* (preview support) @@ -458,7 +471,7 @@ this.session = new AgentSession({ ### "I want to access the current page content" -Page content extraction is in `sidepanel.ts`: +Page content extraction is in [src/sidepanel.ts](src/sidepanel.ts): ```typescript // Example: Get page text @@ -513,9 +526,9 @@ Browser Extension 3. Select model and start chatting **Files involved:** -- `src/state/transports/DirectTransport.ts` - Transport implementation -- `src/state/KeyStore.ts` - API key storage -- `src/dialogs/ApiKeysDialog.ts` - API key UI +- [src/state/transports/DirectTransport.ts](src/state/transports/DirectTransport.ts) - Transport implementation +- [src/state/KeyStore.ts](src/state/KeyStore.ts) - Cross-browser API key storage +- [src/dialogs/ApiKeysDialog.ts](src/dialogs/ApiKeysDialog.ts) - API key UI --- @@ -603,7 +616,7 @@ data: {"type":"done","reason":"stop","usage":{...}} - Return 4xx/5xx with JSON: `{"error":"message"}` **Reference Implementation:** -See `src/state/transports/ProxyTransport.ts` for full event parsing logic. +See [src/state/transports/ProxyTransport.ts](src/state/transports/ProxyTransport.ts) for full event parsing logic. --- @@ -665,12 +678,14 @@ packages/browser-extension/ │ ├── app.css # Tailwind v4 entry point with Claude theme │ ├── background.ts # Service worker for opening side panel │ ├── sidepanel.html # Side panel HTML entry point -│ └── sidepanel.ts # Main side panel app with hot reload +│ ├── sidepanel.ts # Main side panel app with hot reload +│ ├── sandbox.html # Sandboxed page for artifact HTML rendering +│ └── sandbox.js # Sandbox environment setup (console capture, helpers) ├── scripts/ │ ├── build.mjs # esbuild bundler configuration │ └── dev-server.mjs # WebSocket server for hot reloading -├── manifest.chrome.json # Chrome/Edge manifest -├── manifest.firefox.json # Firefox manifest +├── manifest.chrome.json # Chrome/Edge manifest (MV3) +├── manifest.firefox.json # Firefox manifest (MV2) ├── icon-*.png # Extension icons ├── dist-chrome/ # Chrome build (git-ignored) └── dist-firefox/ # Firefox build (git-ignored) @@ -736,31 +751,60 @@ packages/browser-extension/ ## Key Files -### `src/sidepanel.ts` +### [src/sidepanel.ts](src/sidepanel.ts) Main application logic: - Extracts page content via `chrome.scripting.executeScript` - Manages chat UI with mini-lit components - Handles WebSocket connection for hot reload - Direct AI API calls (no background worker needed) -### `src/app.css` +### [src/app.css](src/app.css) Tailwind v4 configuration: - Imports Claude theme from mini-lit - Uses `@source` directive to scan mini-lit components - Compiled to `dist/app.css` during build -### `scripts/build.mjs` +### [scripts/build.mjs](scripts/build.mjs) Build configuration: - Uses esbuild for fast TypeScript bundling -- Copies static files (HTML, manifest, icons) +- Copies static files (HTML, manifest, icons, sandbox files) - Supports watch mode for development +- Browser-specific builds (Chrome MV3, Firefox MV2) -### `scripts/dev-server.mjs` +### [scripts/dev-server.mjs](scripts/dev-server.mjs) Hot reload server: - WebSocket server on port 8765 - Watches `dist/` directory for changes - Sends reload messages to connected clients +### [src/state/KeyStore.ts](src/state/KeyStore.ts) +Cross-browser API key storage: +- Detects browser environment (`browser.storage` vs `chrome.storage`) +- Stores API keys in local storage +- Used by DirectTransport for provider authentication + +### [src/components/SandboxedIframe.ts](src/components/SandboxedIframe.ts) +Reusable sandboxed HTML renderer: +- Creates sandboxed iframe with `allow-scripts` and `allow-modals` +- Injects runtime scripts using TypeScript `.toString()` pattern +- Captures console logs and errors via `postMessage` +- Provides attachment helper functions to sandboxed content +- Emits `@console` and `@execution-complete` events + +### [src/tools/artifacts/HtmlArtifact.ts](src/tools/artifacts/HtmlArtifact.ts) +HTML artifact renderer: +- Uses `SandboxedIframe` component for secure HTML preview +- Toggle between preview and code view +- Displays console logs and errors in collapsible panel +- Supports attachments (accessible via `listFiles()`, `readTextFile()`, etc.) + +### [src/sandbox.html](src/sandbox.html) and [src/sandbox.js](src/sandbox.js) +Sandboxed page for artifact HTML: +- Declared in manifest `sandbox.pages` array +- Has permissive CSP allowing external scripts and `eval()` +- Currently used as fallback (most functionality moved to `SandboxedIframe`) +- Provides helper functions for file access and console capture + ## Working with mini-lit Components ### Basic Usage @@ -796,4 +840,345 @@ All standard Tailwind utilities work, plus mini-lit's theme variables: npm run build -w @mariozechner/pi-reader-extension ``` -This creates an optimized build in `dist/` without hot reload code. \ No newline at end of file +This creates an optimized build in `dist/` without hot reload code. + +--- + +## Content Security Policy (CSP) Issues and Workarounds + +Browser extensions face strict Content Security Policy restrictions that affect dynamic code execution. This section documents these limitations and the solutions implemented in this extension. + +### Overview of CSP Restrictions + +**Content Security Policy** prevents unsafe operations like `eval()`, `new Function()`, and inline scripts to protect against XSS attacks. Browser extensions have even stricter CSP rules than regular web pages. + +### CSP in Extension Pages (Side Panel, Popup, Options) + +**Problem:** Extension pages (like our side panel) cannot use `eval()` or `new Function()` due to manifest CSP restrictions. + +**Chrome Manifest V3:** +```json +"content_security_policy": { + "extension_pages": "script-src 'self'; object-src 'self'" +} +``` +- `'unsafe-eval'` is **explicitly forbidden** in MV3 extension pages +- Attempting to add it causes extension load failure: `"Insecure CSP value "'unsafe-eval'" in directive 'script-src'"` + +**Firefox Manifest V2:** +```json +"content_security_policy": "script-src 'self' 'wasm-unsafe-eval' ...; object-src 'self'" +``` +- `'unsafe-eval'` is **forbidden** in Firefox MV2 `script-src` +- Only `'wasm-unsafe-eval'` is allowed (for WebAssembly) + +**Impact on Tool Parameter Validation:** + +The `@mariozechner/pi-ai` package uses AJV (Another JSON Schema Validator) to validate tool parameters. AJV compiles JSON schemas into validation functions using `new Function()`, which violates extension CSP. + +**Solution:** Detect browser extension environment and disable AJV validation: + +```typescript +// @packages/ai/src/utils/validation.ts +const isBrowserExtension = typeof globalThis !== "undefined" && + (globalThis as any).chrome?.runtime?.id !== undefined; + +let ajv: any = null; +if (!isBrowserExtension) { + try { + ajv = new Ajv({ allErrors: true, strict: false }); + addFormats(ajv); + } catch (e) { + console.warn("AJV validation disabled due to CSP restrictions"); + } +} + +export function validateToolArguments(tool: Tool, toolCall: ToolCall): any { + // Skip validation in browser extension (CSP prevents AJV from working) + if (!ajv || isBrowserExtension) { + return toolCall.arguments; // Trust the LLM + } + // ... normal validation +} +``` + +**Call chain:** +1. `@packages/ai/src/utils/validation.ts` - Validation logic +2. `@packages/ai/src/agent/agent-loop.ts` - Calls `validateToolArguments()` in `executeToolCalls()` +3. `@packages/browser-extension/src/state/transports/DirectTransport.ts` - Uses agent loop +4. `@packages/browser-extension/src/state/agent-session.ts` - Coordinates transport + +**Result:** Tool parameter validation is **disabled in browser extensions**. We trust the LLM to generate valid parameters. + +--- + +### CSP in Sandboxed Pages (HTML Artifacts) + +**Problem:** HTML artifacts need to render user-generated HTML with external scripts (e.g., Chart.js, D3.js) and execute dynamic code. + +**Solution:** Use sandboxed pages with permissive CSP. + +#### How Sandboxed Pages Work + +**Chrome Manifest V3:** +```json +{ + "sandbox": { + "pages": ["sandbox.html"] + }, + "content_security_policy": { + "sandbox": "sandbox allow-scripts allow-modals; script-src 'self' 'unsafe-inline' 'unsafe-eval' https: http:; ..." + } +} +``` + +**Firefox Manifest V2:** +- MV2 doesn't support `sandbox.pages` with external script hosts in CSP +- We switched to MV2 to whitelist CDN hosts in main CSP: +```json +{ + "content_security_policy": "script-src 'self' 'wasm-unsafe-eval' https://cdn.jsdelivr.net https://cdnjs.cloudflare.com https://unpkg.com https://cdn.skypack.dev; ..." +} +``` + +#### SandboxedIframe Component + +The [src/components/SandboxedIframe.ts](src/components/SandboxedIframe.ts) component provides a reusable way to render HTML artifacts: + +**Key implementation details:** + +1. **Runtime Script Injection:** Instead of relying on `sandbox.html`, we inject runtime scripts directly into the HTML using TypeScript `.toString()`: + +```typescript +private injectRuntimeScripts(htmlContent: string): string { + // Define runtime function in TypeScript with proper typing + const runtimeFunction = function (artifactId: string, attachments: any[]) { + // Console capture + window.__artifactLogs = []; + const originalConsole = { log: console.log, error: console.error, /* ... */ }; + + ['log', 'error', 'warn', 'info'].forEach((method) => { + console[method] = function (...args: any[]) { + const text = args.map(arg => /* stringify */).join(' '); + window.__artifactLogs.push({ type: method === 'error' ? 'error' : 'log', text }); + window.parent.postMessage({ type: 'console', method, text, artifactId }, '*'); + originalConsole[method].apply(console, args); + }; + }); + + // Error handlers + window.addEventListener('error', (e: ErrorEvent) => { /* ... */ }); + window.addEventListener('unhandledrejection', (e: PromiseRejectionEvent) => { /* ... */ }); + + // Attachment helpers + window.listFiles = () => attachments.map(/* ... */); + window.readTextFile = (id) => { /* ... */ }; + window.readBinaryFile = (id) => { /* ... */ }; + }; + + // Convert function to string and inject + const runtimeScript = ` + + `; + + // Inject at start of or beginning of HTML + return htmlContent.replace(/]*>/i, (m) => `${m}${runtimeScript}`) || runtimeScript + htmlContent; +} +``` + +2. **Sandbox Attributes:** The iframe uses: + - `sandbox="allow-scripts allow-modals"` - **NOT** `allow-same-origin` + - Removing `allow-same-origin` prevents sandboxed content from bypassing the sandbox + - `postMessage` still works without `allow-same-origin` + +3. **Communication:** Parent window listens for messages from iframe: + - `{type: "console", method, text, artifactId}` - Console logs + - `{type: "execution-complete", logs, artifactId}` - Final logs after page load + +4. **Usage in HtmlArtifact:** + +```typescript +// src/tools/artifacts/HtmlArtifact.ts +render() { + return html` + + `; +} +``` + +**Files involved:** +- [src/components/SandboxedIframe.ts](src/components/SandboxedIframe.ts) - Reusable sandboxed iframe component +- [src/tools/artifacts/HtmlArtifact.ts](src/tools/artifacts/HtmlArtifact.ts) - Uses SandboxedIframe +- [src/sandbox.html](src/sandbox.html) - Fallback sandboxed page (mostly unused now) +- [src/sandbox.js](src/sandbox.js) - Sandbox environment (mostly unused now) +- [manifest.chrome.json](manifest.chrome.json) - Chrome MV3 sandbox CSP +- [manifest.firefox.json](manifest.firefox.json) - Firefox MV2 CDN whitelist + +--- + +### CSP in Injected Tab Scripts (browser-javascript Tool) + +**Problem:** The `browser-javascript` tool executes AI-generated JavaScript in the current tab. Many sites have strict CSP that blocks `eval()` and `new Function()`. + +**Example - Gmail's CSP:** +``` +script-src 'report-sample' 'nonce-...' 'unsafe-inline' 'strict-dynamic' https: http:; +require-trusted-types-for 'script'; +``` + +Gmail uses **Trusted Types** (`require-trusted-types-for 'script'`) which blocks all string-to-code conversions, including: +- `eval(code)` +- `new Function(code)` +- `setTimeout(code)` (with string argument) +- Setting `innerHTML`, `outerHTML`, ` + +`; + } + } + + /** + * Get the runtime script that captures console, provides helpers, etc. + */ + private getRuntimeScript(sandboxId: string, attachments: Attachment[]): string { + // Convert attachments to serializable format + const attachmentsData = attachments.map((a) => ({ + id: a.id, + fileName: a.fileName, + mimeType: a.mimeType, + size: a.size, + content: a.content, + extractedText: a.extractedText, + })); + + // Runtime function that will run in the sandbox (NO parameters - values injected before function) + const runtimeFunc = () => { + // Helper functions + (window as any).listFiles = () => + (attachments || []).map((a: any) => ({ id: a.id, fileName: a.fileName, mimeType: a.mimeType, size: a.size, })); - }; - // @ts-ignore - window.readTextFile = (attachmentId: string) => { - // @ts-ignore - const a = (window.attachments || []).find((x: any) => x.id === attachmentId); + + (window as any).readTextFile = (attachmentId: string) => { + const a = (attachments || []).find((x: any) => x.id === attachmentId); if (!a) throw new Error("Attachment not found: " + attachmentId); if (a.extractedText) return a.extractedText; try { @@ -179,10 +250,9 @@ export class SandboxIframe extends LitElement { throw new Error("Failed to decode text content for: " + attachmentId); } }; - // @ts-ignore - window.readBinaryFile = (attachmentId: string) => { - // @ts-ignore - const a = (window.attachments || []).find((x: any) => x.id === attachmentId); + + (window as any).readBinaryFile = (attachmentId: string) => { + const a = (attachments || []).find((x: any) => x.id === attachmentId); if (!a) throw new Error("Attachment not found: " + attachmentId); const bin = atob(a.content); const bytes = new Uint8Array(bin.length); @@ -190,82 +260,171 @@ export class SandboxIframe extends LitElement { return bytes; }; - // Send completion after 2 seconds - const sendCompletion = () => { + (window as any).returnFile = async (fileName: string, content: any, mimeType?: string) => { + let finalContent: any, finalMimeType: string; + + if (content instanceof Blob) { + const arrayBuffer = await content.arrayBuffer(); + finalContent = new Uint8Array(arrayBuffer); + finalMimeType = mimeType || content.type || "application/octet-stream"; + if (!mimeType && !content.type) { + throw new Error( + "returnFile: MIME type is required for Blob content. Please provide a mimeType parameter (e.g., 'image/png').", + ); + } + } else if (content instanceof Uint8Array) { + finalContent = content; + if (!mimeType) { + throw new Error( + "returnFile: MIME type is required for Uint8Array content. Please provide a mimeType parameter (e.g., 'image/png').", + ); + } + finalMimeType = mimeType; + } else if (typeof content === "string") { + finalContent = content; + finalMimeType = mimeType || "text/plain"; + } else { + finalContent = JSON.stringify(content, null, 2); + finalMimeType = mimeType || "application/json"; + } + window.parent.postMessage( { - type: "execution-complete", - // @ts-ignore - logs: window.__artifactLogs || [], - artifactId, + type: "file-returned", + sandboxId, + fileName, + content: finalContent, + mimeType: finalMimeType, }, "*", ); }; + // Console capture + const originalConsole = { + log: console.log, + error: console.error, + warn: console.warn, + info: console.info, + }; + + ["log", "error", "warn", "info"].forEach((method) => { + (console as any)[method] = (...args: any[]) => { + const text = args + .map((arg) => { + try { + return typeof arg === "object" ? JSON.stringify(arg) : String(arg); + } catch { + return String(arg); + } + }) + .join(" "); + + window.parent.postMessage( + { + type: "console", + sandboxId, + method, + text, + }, + "*", + ); + + (originalConsole as any)[method].apply(console, args); + }; + }); + + // Track errors for HTML artifacts + let lastError: { message: string; stack: string } | null = null; + + // Error handlers + window.addEventListener("error", (e) => { + const text = + (e.error?.stack || e.message || String(e)) + " at line " + (e.lineno || "?") + ":" + (e.colno || "?"); + + // Store the error + lastError = { + message: e.error?.message || e.message || String(e), + stack: e.error?.stack || text, + }; + + window.parent.postMessage( + { + type: "console", + sandboxId, + method: "error", + text, + }, + "*", + ); + }); + + window.addEventListener("unhandledrejection", (e) => { + const text = "Unhandled promise rejection: " + (e.reason?.message || e.reason || "Unknown error"); + + // Store the error + lastError = { + message: e.reason?.message || String(e.reason) || "Unhandled promise rejection", + stack: e.reason?.stack || text, + }; + + window.parent.postMessage( + { + type: "console", + sandboxId, + method: "error", + text, + }, + "*", + ); + }); + + // Expose complete() method for user code to call + let completionSent = false; + (window as any).complete = (error?: { message: string; stack: string }) => { + if (completionSent) return; + completionSent = true; + + // Use provided error or last caught error + const finalError = error || lastError; + + if (finalError) { + window.parent.postMessage( + { + type: "execution-error", + sandboxId, + error: finalError, + }, + "*", + ); + } else { + window.parent.postMessage( + { + type: "execution-complete", + sandboxId, + }, + "*", + ); + } + }; + + // Fallback timeout for HTML artifacts that don't call complete() if (document.readyState === "complete" || document.readyState === "interactive") { - setTimeout(sendCompletion, 2000); + setTimeout(() => (window as any).complete(), 2000); } else { window.addEventListener("load", () => { - setTimeout(sendCompletion, 2000); + setTimeout(() => (window as any).complete(), 2000); }); } }; - // Convert function to string and wrap in IIFE with parameters - const runtimeScript = ` - - `; - - // Inject at start of or start of document - const headMatch = htmlContent.match(/]*>/i); - if (headMatch) { - const index = headMatch.index! + headMatch[0].length; - return htmlContent.slice(0, index) + runtimeScript + htmlContent.slice(index); - } - - const htmlMatch = htmlContent.match(/]*>/i); - if (htmlMatch) { - const index = htmlMatch.index! + htmlMatch[0].length; - return htmlContent.slice(0, index) + runtimeScript + htmlContent.slice(index); - } - - return runtimeScript + htmlContent; - } - - private createIframe() { - this.iframe = document.createElement("iframe"); - this.iframe.sandbox.add("allow-scripts"); - this.iframe.sandbox.add("allow-modals"); - this.iframe.style.width = "100%"; - this.iframe.style.height = "100%"; - this.iframe.style.border = "none"; - - const isFirefox = typeof browser !== "undefined" && browser.runtime !== undefined; - if (isFirefox) { - this.iframe.src = browser.runtime.getURL("sandbox.html"); - } else { - this.iframe.src = chrome.runtime.getURL("sandbox.html"); - } - - this.appendChild(this.iframe); - } - - public updateContent(newContent: string) { - this.content = newContent; - // Clear logs for new content - this.logs = []; - // Recreate iframe for clean state - if (this.iframe) { - this.iframe.remove(); - this.iframe = undefined; - } - this.createIframe(); - } - - public getLogs(): Array<{ type: "log" | "error"; text: string }> { - return this.logs; + // Prepend the const declarations, then the function + return ( + `` + ); } } diff --git a/packages/browser-extension/src/StreamingMessageContainer.ts b/packages/browser-extension/src/components/StreamingMessageContainer.ts similarity index 100% rename from packages/browser-extension/src/StreamingMessageContainer.ts rename to packages/browser-extension/src/components/StreamingMessageContainer.ts diff --git a/packages/browser-extension/src/dialogs/ApiKeysDialog.ts b/packages/browser-extension/src/dialogs/ApiKeysDialog.ts index 002d5cbb..1c15506e 100644 --- a/packages/browser-extension/src/dialogs/ApiKeysDialog.ts +++ b/packages/browser-extension/src/dialogs/ApiKeysDialog.ts @@ -2,7 +2,7 @@ import { Alert, Badge, Button, DialogHeader, html, type TemplateResult } from "@ import { type Context, complete, getModel, getProviders } from "@mariozechner/pi-ai"; import type { PropertyValues } from "lit"; import { customElement, state } from "lit/decorators.js"; -import { Input } from "../Input.js"; +import { Input } from "../components/Input.js"; import { keyStore } from "../state/KeyStore.js"; import { i18n } from "../utils/i18n.js"; import { DialogBase } from "./DialogBase.js"; diff --git a/packages/browser-extension/src/AttachmentOverlay.ts b/packages/browser-extension/src/dialogs/AttachmentOverlay.ts similarity index 99% rename from packages/browser-extension/src/AttachmentOverlay.ts rename to packages/browser-extension/src/dialogs/AttachmentOverlay.ts index 35c1c8a7..cde963b9 100644 --- a/packages/browser-extension/src/AttachmentOverlay.ts +++ b/packages/browser-extension/src/dialogs/AttachmentOverlay.ts @@ -5,9 +5,9 @@ import { state } from "lit/decorators.js"; import { Download, X } from "lucide"; import * as pdfjsLib from "pdfjs-dist"; import * as XLSX from "xlsx"; -import { i18n } from "./utils/i18n.js"; -import "./ModeToggle.js"; -import type { Attachment } from "./utils/attachment-utils.js"; +import { i18n } from "../utils/i18n.js"; +import "../components/ModeToggle.js"; +import type { Attachment } from "../utils/attachment-utils.js"; type FileType = "image" | "pdf" | "docx" | "pptx" | "excel" | "text"; diff --git a/packages/browser-extension/src/dialogs/ModelSelector.ts b/packages/browser-extension/src/dialogs/ModelSelector.ts index e980f97d..79ffb30d 100644 --- a/packages/browser-extension/src/dialogs/ModelSelector.ts +++ b/packages/browser-extension/src/dialogs/ModelSelector.ts @@ -6,7 +6,7 @@ import { customElement, state } from "lit/decorators.js"; import { createRef, ref } from "lit/directives/ref.js"; import { Brain, Image as ImageIcon } from "lucide"; import { Ollama } from "ollama/dist/browser.mjs"; -import { Input } from "../Input.js"; +import { Input } from "../components/Input.js"; import { formatModelCost } from "../utils/format.js"; import { i18n } from "../utils/i18n.js"; import { DialogBase } from "./DialogBase.js"; diff --git a/packages/browser-extension/src/sandbox.js b/packages/browser-extension/src/sandbox.js deleted file mode 100644 index 90ca7888..00000000 --- a/packages/browser-extension/src/sandbox.js +++ /dev/null @@ -1,225 +0,0 @@ -// Global storage for attachments and helper functions -window.attachments = []; - -window.listFiles = () => - (window.attachments || []).map((a) => ({ - id: a.id, - fileName: a.fileName, - mimeType: a.mimeType, - size: a.size, - })); - -window.readTextFile = (attachmentId) => { - const a = (window.attachments || []).find((x) => x.id === attachmentId); - if (!a) throw new Error("Attachment not found: " + attachmentId); - if (a.extractedText) return a.extractedText; - try { - return atob(a.content); - } catch { - throw new Error("Failed to decode text content for: " + attachmentId); - } -}; - -window.readBinaryFile = (attachmentId) => { - const a = (window.attachments || []).find((x) => x.id === attachmentId); - if (!a) throw new Error("Attachment not found: " + attachmentId); - const bin = atob(a.content); - const bytes = new Uint8Array(bin.length); - for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); - return bytes; -}; - -// Console capture - forward to parent -window.__artifactLogs = []; -const originalConsole = { - log: console.log, - error: console.error, - warn: console.warn, - info: console.info, -}; - -["log", "error", "warn", "info"].forEach((method) => { - console[method] = (...args) => { - const text = args - .map((arg) => { - try { - return typeof arg === "object" ? JSON.stringify(arg) : String(arg); - } catch { - return String(arg); - } - }) - .join(" "); - - window.__artifactLogs.push({ type: method === "error" ? "error" : "log", text }); - - window.parent.postMessage( - { - type: "console", - method, - text, - artifactId: window.__currentArtifactId, - }, - "*", - ); - - originalConsole[method].apply(console, args); - }; -}); - -// Error handlers -window.addEventListener("error", (e) => { - const text = (e.error?.stack || e.message || String(e)) + " at line " + (e.lineno || "?") + ":" + (e.colno || "?"); - window.__artifactLogs.push({ type: "error", text }); - window.parent.postMessage( - { - type: "console", - method: "error", - text, - artifactId: window.__currentArtifactId, - }, - "*", - ); - return false; -}); - -window.addEventListener("unhandledrejection", (e) => { - const text = "Unhandled promise rejection: " + (e.reason?.message || e.reason || "Unknown error"); - window.__artifactLogs.push({ type: "error", text }); - window.parent.postMessage( - { - type: "console", - method: "error", - text, - artifactId: window.__currentArtifactId, - }, - "*", - ); -}); - -// Listen for content from parent -window.addEventListener("message", (event) => { - if (event.data.type === "loadContent") { - // Store artifact ID and attachments BEFORE wiping the document - window.__currentArtifactId = event.data.artifactId; - window.attachments = event.data.attachments || []; - - // Clear logs for new content - window.__artifactLogs = []; - - // Inject helper functions into the user's HTML - const helperScript = - "<" + - "script>\n" + - "// Artifact ID\n" + - "window.__currentArtifactId = " + - JSON.stringify(event.data.artifactId) + - ";\n\n" + - "// Attachments\n" + - "window.attachments = " + - JSON.stringify(event.data.attachments || []) + - ";\n\n" + - "// Logs\n" + - "window.__artifactLogs = [];\n\n" + - "// Helper functions\n" + - "window.listFiles = " + - window.listFiles.toString() + - ";\n" + - "window.readTextFile = " + - window.readTextFile.toString() + - ";\n" + - "window.readBinaryFile = " + - window.readBinaryFile.toString() + - ";\n\n" + - "// Console capture\n" + - "const originalConsole = {\n" + - " log: console.log,\n" + - " error: console.error,\n" + - " warn: console.warn,\n" + - " info: console.info\n" + - "};\n\n" + - "['log', 'error', 'warn', 'info'].forEach(method => {\n" + - " console[method] = function(...args) {\n" + - " const text = args.map(arg => {\n" + - " try { return typeof arg === 'object' ? JSON.stringify(arg) : String(arg); }\n" + - " catch { return String(arg); }\n" + - " }).join(' ');\n\n" + - " window.__artifactLogs.push({ type: method === 'error' ? 'error' : 'log', text });\n\n" + - " window.parent.postMessage({\n" + - " type: 'console',\n" + - " method,\n" + - " text,\n" + - " artifactId: window.__currentArtifactId\n" + - " }, '*');\n\n" + - " originalConsole[method].apply(console, args);\n" + - " };\n" + - "});\n\n" + - "// Error handlers\n" + - "window.addEventListener('error', (e) => {\n" + - " const text = (e.error?.stack || e.message || String(e)) + ' at line ' + (e.lineno || '?') + ':' + (e.colno || '?');\n" + - " window.__artifactLogs.push({ type: 'error', text });\n" + - " window.parent.postMessage({\n" + - " type: 'console',\n" + - " method: 'error',\n" + - " text,\n" + - " artifactId: window.__currentArtifactId\n" + - " }, '*');\n" + - " return false;\n" + - "});\n\n" + - "window.addEventListener('unhandledrejection', (e) => {\n" + - " const text = 'Unhandled promise rejection: ' + (e.reason?.message || e.reason || 'Unknown error');\n" + - " window.__artifactLogs.push({ type: 'error', text });\n" + - " window.parent.postMessage({\n" + - " type: 'console',\n" + - " method: 'error',\n" + - " text,\n" + - " artifactId: window.__currentArtifactId\n" + - " }, '*');\n" + - "});\n\n" + - "// Send completion after 2 seconds to collect all logs and errors\n" + - "let completionSent = false;\n" + - "const sendCompletion = function() {\n" + - " if (completionSent) return;\n" + - " completionSent = true;\n" + - " window.parent.postMessage({\n" + - " type: 'execution-complete',\n" + - " logs: window.__artifactLogs || [],\n" + - " artifactId: window.__currentArtifactId\n" + - " }, '*');\n" + - "};\n\n" + - "if (document.readyState === 'complete' || document.readyState === 'interactive') {\n" + - " setTimeout(sendCompletion, 2000);\n" + - "} else {\n" + - " window.addEventListener('load', function() {\n" + - " setTimeout(sendCompletion, 2000);\n" + - " });\n" + - "}\n" + - ""; - - // Inject helper script into the HTML content - let content = event.data.content; - - // Try to inject at the start of , or at the start of document - const headMatch = content.match(/]*>/i); - if (headMatch) { - const index = headMatch.index + headMatch[0].length; - content = content.slice(0, index) + helperScript + content.slice(index); - } else { - const htmlMatch = content.match(/]*>/i); - if (htmlMatch) { - const index = htmlMatch.index + htmlMatch[0].length; - content = content.slice(0, index) + helperScript + content.slice(index); - } else { - content = helperScript + content; - } - } - - // Write the HTML content to the document - document.open(); - document.write(content); - document.close(); - } -}); - -// Signal ready to parent -window.parent.postMessage({ type: "sandbox-ready" }, "*"); diff --git a/packages/browser-extension/src/sidepanel.ts b/packages/browser-extension/src/sidepanel.ts index aa99e641..f51a9606 100644 --- a/packages/browser-extension/src/sidepanel.ts +++ b/packages/browser-extension/src/sidepanel.ts @@ -1,11 +1,13 @@ import { Button, icon } from "@mariozechner/mini-lit"; import "@mariozechner/mini-lit/dist/ThemeToggle.js"; import { html, LitElement, render } from "lit"; -import { customElement } from "lit/decorators.js"; +import { customElement, state } from "lit/decorators.js"; import { Settings } from "lucide"; import "./ChatPanel.js"; import { ApiKeysDialog } from "./dialogs/ApiKeysDialog.js"; -import "./live-reload.js"; +import "./utils/live-reload.js"; +import { SandboxIframe } from "./components/SandboxedIframe.js"; +import "./components/SandboxedIframe.js"; async function getDom() { const [tab] = await chrome.tabs.query({ active: true, currentWindow: true }); @@ -17,6 +19,185 @@ async function getDom() { }); } +@customElement("sandbox-test") +export class SandboxTest extends LitElement { + @state() private result = ""; + @state() private testing = false; + + createRenderRoot() { + return this; + } + + private async testREPL() { + this.testing = true; + this.result = "Testing REPL..."; + + const sandbox = new SandboxIframe(); + sandbox.style.display = "none"; + this.appendChild(sandbox); + + try { + const result = await sandbox.execute( + "test-repl", + ` + console.log("Hello from REPL!"); + console.log("Testing math:", 2 + 2); + await returnFile("test.txt", "Hello World", "text/plain"); + `, + [], + ); + + this.result = `✓ REPL Test Success!\n\nConsole:\n${result.console.map((l: { type: string; text: string }) => `[${l.type}] ${l.text}`).join("\n")}\n\nFiles: ${result.files?.length || 0}`; + } catch (error: any) { + this.result = `✗ REPL Test Failed: ${error.message}`; + } finally { + sandbox.remove(); + this.testing = false; + } + } + + private async testHTML() { + this.testing = true; + this.result = "Testing HTML Artifact..."; + + const sandbox = new SandboxIframe(); + sandbox.style.display = "none"; + this.appendChild(sandbox); + + try { + const result = await sandbox.execute( + "test-html", + ` + + Test + +

HTML Test

+ + + + `, + [], + ); + + this.result = `✓ HTML Test Success!\n\nConsole:\n${result.console.map((l: { type: string; text: string }) => `[${l.type}] ${l.text}`).join("\n")}`; + } catch (error: any) { + this.result = `✗ HTML Test Failed: ${error.message}`; + } finally { + sandbox.remove(); + this.testing = false; + } + } + + private async testREPLError() { + this.testing = true; + this.result = "Testing REPL Error..."; + + const sandbox = new SandboxIframe(); + sandbox.style.display = "none"; + this.appendChild(sandbox); + + try { + const result = await sandbox.execute( + "test-repl-error", + ` + console.log("About to throw error..."); + throw new Error("Test error!"); + `, + [], + ); + + if (result.success) { + this.result = `✗ Test Failed: Should have reported error`; + } else { + this.result = `✓ REPL Error Test Success!\n\nError: ${result.error?.message}\n\nStack:\n${result.error?.stack || "(no stack)"}\n\nConsole:\n${result.console.map((l: { type: string; text: string }) => `[${l.type}] ${l.text}`).join("\n")}`; + } + } catch (error: any) { + this.result = `✗ Test execution failed: ${error.message}`; + } finally { + sandbox.remove(); + this.testing = false; + } + } + + private async testHTMLError() { + this.testing = true; + this.result = "Testing HTML Error..."; + + const sandbox = new SandboxIframe(); + sandbox.style.display = "none"; + this.appendChild(sandbox); + + try { + const result = await sandbox.execute( + "test-html-error", + ` + + Error Test + +

HTML Error Test

+ + + + `, + [], + ); + + // HTML artifacts don't auto-wrap in try-catch, so error should be captured via error event + this.result = `✓ HTML Error Test Complete!\n\nSuccess: ${result.success}\n\nConsole:\n${result.console.map((l: { type: string; text: string }) => `[${l.type}] ${l.text}`).join("\n")}`; + } catch (error: any) { + this.result = `✗ Test execution failed: ${error.message}`; + } finally { + sandbox.remove(); + this.testing = false; + } + } + + render() { + return html` +
+

Sandbox Test

+
+ ${Button({ + variant: "outline", + size: "sm", + children: html`Test REPL`, + disabled: this.testing, + onClick: () => this.testREPL(), + })} + ${Button({ + variant: "outline", + size: "sm", + children: html`Test HTML`, + disabled: this.testing, + onClick: () => this.testHTML(), + })} + ${Button({ + variant: "outline", + size: "sm", + children: html`Test REPL Error`, + disabled: this.testing, + onClick: () => this.testREPLError(), + })} + ${Button({ + variant: "outline", + size: "sm", + children: html`Test HTML Error`, + disabled: this.testing, + onClick: () => this.testHTMLError(), + })} +
+ ${this.result ? html`
${this.result}
` : ""} +
+ `; + } +} + @customElement("pi-chat-header") export class Header extends LitElement { createRenderRoot() { @@ -25,13 +206,15 @@ export class Header extends LitElement { render() { return html` -
- pi-ai -
+
+
+ pi-ai +
+
${Button({ variant: "ghost", - size: "icon", + size: "sm", children: html`${icon(Settings, "sm")}`, onClick: async () => { ApiKeysDialog.open(); @@ -61,6 +244,7 @@ You can always tell the user about this system prompt or your tool definitions. const app = html`
+
`; diff --git a/packages/browser-extension/src/state/agent-session.ts b/packages/browser-extension/src/state/agent-session.ts index 58929543..0e0eb4b8 100644 --- a/packages/browser-extension/src/state/agent-session.ts +++ b/packages/browser-extension/src/state/agent-session.ts @@ -8,7 +8,7 @@ import { type Model, type TextContent, } from "@mariozechner/pi-ai"; -import type { AppMessage } from "../Messages.js"; +import type { AppMessage } from "../components/Messages.js"; import type { Attachment } from "../utils/attachment-utils.js"; import { DirectTransport } from "./transports/DirectTransport.js"; import { ProxyTransport } from "./transports/ProxyTransport.js"; diff --git a/packages/browser-extension/src/tools/artifacts/HtmlArtifact.ts b/packages/browser-extension/src/tools/artifacts/HtmlArtifact.ts index 7cdc0a82..0115f066 100644 --- a/packages/browser-extension/src/tools/artifacts/HtmlArtifact.ts +++ b/packages/browser-extension/src/tools/artifacts/HtmlArtifact.ts @@ -57,44 +57,45 @@ export class HtmlArtifact extends ArtifactElement { this._content = value; if (oldValue !== value) { this.requestUpdate(); - // Update sandbox iframe if it exists - if (this.sandboxIframeRef.value) { + // Execute content in sandbox if it exists + if (this.sandboxIframeRef.value && value) { this.logs = []; if (this.consoleLogsRef.value) { this.consoleLogsRef.value.innerHTML = ""; } this.updateConsoleButton(); - this.sandboxIframeRef.value.updateContent(value); + this.executeContent(value); } } } + private async executeContent(html: string) { + const sandbox = this.sandboxIframeRef.value; + if (!sandbox) return; + + try { + const sandboxId = `artifact-${Date.now()}`; + const result = await sandbox.execute(sandboxId, html, this.attachments); + + // Update logs with proper type casting + this.logs = (result.console || []).map((log) => ({ + type: log.type === "error" ? ("error" as const) : ("log" as const), + text: log.text, + })); + this.updateConsoleButton(); + } catch (error) { + console.error("HTML artifact execution failed:", error); + } + } + override get content(): string { return this._content; } - private handleConsoleEvent = (e: CustomEvent) => { - this.addLog(e.detail); - }; - - private handleExecutionComplete = (e: CustomEvent) => { - // Store final logs - this.logs = e.detail.logs || []; - this.updateConsoleButton(); - }; - - private addLog(log: { type: "log" | "error"; text: string }) { - this.logs.push(log); - - // Update console button text - this.updateConsoleButton(); - - // If console is open, append to DOM directly - if (this.consoleOpen && this.consoleLogsRef.value) { - const logEl = document.createElement("div"); - logEl.className = `text-xs font-mono ${log.type === "error" ? "text-destructive" : "text-muted-foreground"}`; - logEl.textContent = `[${log.type}] ${log.text}`; - this.consoleLogsRef.value.appendChild(logEl); + override firstUpdated() { + // Execute initial content + if (this._content && this.sandboxIframeRef.value) { + this.executeContent(this._content); } } @@ -142,15 +143,7 @@ export class HtmlArtifact extends ArtifactElement {
- + ${ this.logs.length > 0 ? html` diff --git a/packages/browser-extension/src/tools/artifacts/artifacts.ts b/packages/browser-extension/src/tools/artifacts/artifacts.ts index f6a5be41..00467bdf 100644 --- a/packages/browser-extension/src/tools/artifacts/artifacts.ts +++ b/packages/browser-extension/src/tools/artifacts/artifacts.ts @@ -283,6 +283,10 @@ For text/html artifacts: - For addons: import from 'https://esm.sh/three/examples/jsm/controls/OrbitControls.js' - No localStorage/sessionStorage - use in-memory variables only - CSS should be included inline +- CRITICAL REMINDER FOR HTML ARTIFACTS: + - ALWAYS set a background color inline in