diff --git a/packages/browser-extension/PLAN.md b/packages/browser-extension/PLAN.md new file mode 100644 index 00000000..78cb01d4 --- /dev/null +++ b/packages/browser-extension/PLAN.md @@ -0,0 +1,987 @@ +# Porting Plan: genai-workshop-new Chat System to Browser Extension + +## Executive Summary + +Port the complete chat interface, message rendering, streaming, tool execution, and transport system from `genai-workshop-new/src/app` to the browser extension. The goal is to provide a full-featured AI chat interface with: + +1. **Multiple transport options**: Direct API calls OR proxy-based calls +2. **Full message rendering**: Text, thinking blocks, tool calls, attachments, images +3. **Streaming support**: Real-time message streaming with proper batching +4. **Tool execution and rendering**: Extensible tool system with custom renderers +5. **Session management**: State management with persistence +6. **Debug capabilities**: Optional debug view for development + +## Current State Analysis + +### Already Ported to Browser Extension +- ✅ `AttachmentTile.ts` - Display attachment thumbnails +- ✅ `AttachmentOverlay.ts` - Full-screen attachment viewer +- ✅ `MessageEditor.ts` - Input field with attachment support +- ✅ `utils/attachment-utils.ts` - PDF, Office, image processing +- ✅ `utils/i18n.ts` - Internationalization +- ✅ `dialogs/ApiKeysDialog.ts` - API key management +- ✅ `dialogs/ModelSelector.ts` - Model selection dialog +- ✅ `state/KeyStore.ts` - API key storage + +### Available in @mariozechner/mini-lit Package +- ✅ `CodeBlock` - Syntax-highlighted code display +- ✅ `MarkdownBlock` - Markdown rendering +- ✅ `Button`, `Input`, `Select`, `Textarea`, etc. - UI components +- ✅ `ThemeToggle` - Dark/light mode +- ✅ `Dialog` - Base dialog component + +### Needs to Be Ported + +#### Core Chat System +1. **AgentInterface.ts** (325 lines) + - Main chat interface container + - Manages scrolling, auto-scroll behavior + - Coordinates MessageList, StreamingMessageContainer, MessageEditor + - Displays usage stats + - Handles session lifecycle + +2. **MessageList.ts** (78 lines) + - Renders stable (non-streaming) messages + - Uses `repeat()` directive for efficient rendering + - Maps tool results by call ID + - Renders user and assistant messages + +3. **Messages.ts** (286 lines) + - **UserMessage component**: Displays user messages with attachments + - **AssistantMessage component**: Displays assistant messages with text, thinking, tool calls + - **ToolMessage component**: Displays individual tool invocations with debug view + - **ToolMessageDebugView component**: Shows tool call args and results + - **AbortedMessage component**: Shows aborted requests + +4. **StreamingMessageContainer.ts** (95 lines) + - Manages streaming message updates + - Batches updates using `requestAnimationFrame` for performance + - Shows loading indicator during streaming + - Handles immediate updates for clearing + +5. **ConsoleBlock.ts** (62 lines) + - Console-style output display + - Auto-scrolling to bottom + - Copy button functionality + - Used by tool renderers + +#### State Management + +6. **state/agent-session.ts** (282 lines) + - **AgentSession class**: Core state management + - Manages conversation state: messages, model, tools, system prompt, thinking level + - Implements pub/sub pattern for state updates + - Handles message preprocessing (e.g., extracting text from documents) + - Coordinates transport for sending messages + - Collects debug information + - Event types: `state-update`, `error-no-model`, `error-no-api-key` + - Methods: + - `prompt(input, attachments)` - Send user message + - `setModel()`, `setSystemPrompt()`, `setThinkingLevel()`, `setTools()` + - `appendMessage()`, `replaceMessages()`, `clearMessages()` + - `abort()` - Cancel ongoing request + - `subscribe(fn)` - Listen to state changes + +7. **state/session-store.ts** (needs investigation) + - Session persistence to IndexedDB + - Load/save conversation history + - Multiple session management + +#### Transport Layer + +8. **state/transports/types.ts** (17 lines) + - `AgentTransport` interface + - `AgentRunConfig` interface + - Defines contract for transport implementations + +9. **state/transports/proxy-transport.ts** (54 lines) + - **LocalTransport class** (misleadingly named - actually proxy) + - Calls proxy server via `streamSimpleProxy` + - Passes auth token from KeyStore + - Yields events from `agentLoop()` + +10. **NEW: state/transports/direct-transport.ts** (needs creation) + - **DirectTransport class** + - Calls provider APIs directly using API keys from KeyStore + - Uses `@mariozechner/pi-ai`'s `agentLoop()` directly + - No auth token needed + +11. **utils/proxy-client.ts** (285 lines) + - `streamSimpleProxy()` function + - Fetches from `/api/stream` endpoint + - Parses SSE (Server-Sent Events) stream + - Reconstructs partial messages from delta events + - Handles abort signals + - Maps proxy events to `AssistantMessageEvent` + - Detects unauthorized and clears auth token + +12. **NEW: utils/config.ts** (needs creation) + - Transport configuration + - Proxy URL configuration + - Storage key: `transport-mode` ("direct" | "proxy") + - Storage key: `proxy-url` (default: configurable) + +#### Tool System + +13. **tools/index.ts** (40 lines) + - Exports tool functions from `@mariozechner/pi-ai` + - Registers default tool renderers + - Exports `renderToolParams()` and `renderToolResult()` + - Re-exports tool implementations + +14. **tools/types.ts** (needs investigation) + - `ToolRenderer` interface + - Contracts for custom tool renderers + +15. **tools/renderer-registry.ts** (19 lines) + - Global registry: `Map` + - `registerToolRenderer(name, renderer)` function + - `getToolRenderer(name)` function + +16. **tools/renderers/DefaultRenderer.ts** (1162 chars) + - Fallback renderer for unknown tools + - Renders params as JSON + - Renders results as JSON or text + +17. **tools/renderers/CalculateRenderer.ts** (1677 chars) + - Custom renderer for calculate tool + - Shows expression and result + +18. **tools/renderers/GetCurrentTimeRenderer.ts** (1328 chars) + - Custom renderer for time tool + - Shows timezone and formatted time + +19. **tools/renderers/BashRenderer.ts** (1500 chars) + - Custom renderer for bash tool + - Uses ConsoleBlock for output + +20. **tools/javascript-repl.ts** (needs investigation) + - JavaScript REPL tool implementation + - May need adaptation for browser environment + +21. **tools/web-search.ts** (needs investigation) + - Web search tool implementation + - Check if compatible with browser extension + +22. **tools/sleep.ts** (needs investigation) + - Simple sleep/delay tool + +#### Utilities + +23. **utils/format.ts** (needs investigation) + - `formatUsage()` - Format token usage and costs + - Other formatting utilities + +24. **utils/auth-token.ts** (21 lines) + - `getAuthToken()` - Prompt for proxy auth token + - `clearAuthToken()` - Remove from storage + - Uses PromptDialog for input + +25. **dialogs/PromptDialog.ts** (needs investigation) + - Simple text input dialog + - Used for auth token entry + +#### Debug/Development + +26. **DebugView.ts** (needs investigation) + - Debug panel showing request/response details + - ChatML formatting + - SSE event stream + - Timing information (TTFT, total time) + - Optional feature for development + +#### NOT Needed + +- ❌ `demos/` folder - All demo files (ignore) +- ❌ `mini/` folder - All UI components (use @mariozechner/mini-lit instead) +- ❌ `admin/ProxyAdmin.ts` - Proxy server admin (not needed in extension) +- ❌ `CodeBlock.ts` - Available in @mariozechner/mini-lit +- ❌ `MarkdownBlock.ts` - Available in @mariozechner/mini-lit +- ❌ `ScatterPlot.ts` - Demo visualization +- ❌ `tools/artifacts.ts` - Artifact tool (demo feature) +- ❌ `tools/bash-mcp-server.ts` - MCP integration (not feasible in browser) +- ❌ `AttachmentTileList.ts` - Likely superseded by MessageEditor integration + +--- + +## Detailed Porting Tasks + +### Phase 1: Core Message Rendering (Foundation) + +#### Task 1.1: Port ConsoleBlock +**File**: `src/ConsoleBlock.ts` +**Dependencies**: mini-lit icons +**Actions**: +1. Copy `ConsoleBlock.ts` to browser extension +2. Update imports to use `@mariozechner/mini-lit` +3. Replace icon imports with lucide icons: + - `iconCheckLine` → `Check` + - `iconFileCopy2Line` → `Copy` +4. Update i18n strings: + - Add "console", "Copy output", "Copied!" to i18n.ts + +**Verification**: Render `` + +#### Task 1.2: Port Messages.ts (User, Assistant, Tool Components) +**File**: `src/Messages.ts` +**Dependencies**: ConsoleBlock, formatUsage, tool rendering +**Actions**: +1. Copy `Messages.ts` to browser extension +2. Update imports: + - `Button` from `@mariozechner/mini-lit` + - `formatUsage` from utils + - Icons from lucide (ToolsLine, Loader4Line, BugLine) +3. Add new type: `AppMessage` (already have partial in extension) +4. Components to register: + - `user-message` + - `assistant-message` + - `tool-message` + - `tool-message-debug` + - `aborted-message` +5. Update i18n strings: + - "Error:", "Request aborted", "Call", "Result", "(no result)", "Waiting for tool result…", "Call was aborted; no result." +6. Guard all custom element registrations + +**Verification**: +- Render user message with text and attachments +- Render assistant message with text, thinking, tool calls +- Render tool message in pending, success, error states + +#### Task 1.3: Port MessageList +**File**: `src/MessageList.ts` +**Dependencies**: Messages.ts +**Actions**: +1. Copy `MessageList.ts` to browser extension +2. Update imports +3. Uses `repeat()` directive from lit - ensure it's available +4. Register `message-list` element with guard + +**Verification**: Render a list of mixed user/assistant/tool messages + +#### Task 1.4: Port StreamingMessageContainer +**File**: `src/StreamingMessageContainer.ts` +**Dependencies**: Messages.ts +**Actions**: +1. Copy `StreamingMessageContainer.ts` to browser extension +2. Update imports +3. Register `streaming-message-container` element with guard +4. Test batching behavior with rapid updates + +**Verification**: +- Stream messages update smoothly +- Cursor blinks during streaming +- Immediate clear works correctly + +--- + +### Phase 2: Tool System + +#### Task 2.1: Port Tool Types and Registry +**Files**: `src/tools/types.ts`, `src/tools/renderer-registry.ts` +**Actions**: +1. Read `tools/types.ts` to understand `ToolRenderer` interface +2. Copy both files to `src/tools/` +3. Create registry as singleton + +**Verification**: Can register and retrieve renderers + +#### Task 2.2: Port Tool Renderers +**Files**: All `src/tools/renderers/*.ts` +**Actions**: +1. Copy `DefaultRenderer.ts` +2. Copy `CalculateRenderer.ts` +3. Copy `GetCurrentTimeRenderer.ts` +4. Copy `BashRenderer.ts` +5. Update all to use `@mariozechner/mini-lit` and lucide icons +6. Ensure all use ConsoleBlock where needed + +**Verification**: Test each renderer with sample tool calls + +#### Task 2.3: Port Tool Implementations +**Files**: `src/tools/javascript-repl.ts`, `src/tools/web-search.ts`, `src/tools/sleep.ts` +**Actions**: +1. Read each file to assess browser compatibility +2. Port `sleep.ts` (should be trivial) +3. Port `javascript-repl.ts` - may need `new Function()` or eval +4. Port `web-search.ts` - check if it uses fetch or needs adaptation +5. Update `tools/index.ts` to register all renderers and export tools + +**Verification**: Test each tool execution in browser context + +--- + +### Phase 3: Transport Layer + +#### Task 3.1: Port Transport Types +**File**: `src/state/transports/types.ts` +**Actions**: +1. Copy file to `src/state/transports/` +2. Verify types align with pi-ai package + +**Verification**: Types compile correctly + +#### Task 3.2: Port Proxy Client +**File**: `src/utils/proxy-client.ts` +**Dependencies**: auth-token.ts +**Actions**: +1. Copy `proxy-client.ts` to `src/utils/` +2. Update `streamSimpleProxy()` to use configurable proxy URL +3. Read proxy URL from config (default: user-configurable) +4. Update error messages for i18n +5. Add i18n strings: "Proxy error: {status} {statusText}", "Proxy error: {error}", "Auth token is required for proxy transport" + +**Verification**: Can connect to proxy server with auth token + +#### Task 3.3: Port Proxy Transport +**File**: `src/state/transports/proxy-transport.ts` +**Actions**: +1. Copy file to `src/state/transports/` +2. Rename `LocalTransport` to `ProxyTransport` for clarity +3. Update to use `streamSimpleProxy` from proxy-client +4. Integrate with KeyStore for auth token + +**Verification**: Can send message through proxy + +#### Task 3.4: Create Direct Transport +**File**: `src/state/transports/direct-transport.ts` (NEW) +**Actions**: +1. Create new `DirectTransport` class implementing `AgentTransport` +2. Use `agentLoop()` from `@mariozechner/pi-ai` directly +3. Integrate with KeyStore to get API keys per provider +4. Pass API key in options to `agentLoop()` +5. Handle `no-api-key` errors by triggering ApiKeysDialog + +**Example Implementation**: +```typescript +import { agentLoop, type AgentContext, type PromptConfig, type UserMessage } from "@mariozechner/pi-ai"; +import { keyStore } from "../../KeyStore.js"; +import type { AgentRunConfig, AgentTransport } from "./types.js"; + +export class DirectTransport implements AgentTransport { + constructor( + private readonly getMessages: () => Promise, + ) {} + + async *run(userMessage: Message, cfg: AgentRunConfig, signal?: AbortSignal) { + // Get API key from KeyStore + const apiKey = await keyStore.getKey(cfg.model.provider); + if (!apiKey) { + throw new Error("no-api-key"); + } + + const context: AgentContext = { + systemPrompt: cfg.systemPrompt, + messages: await this.getMessages(), + tools: cfg.tools, + }; + + const pc: PromptConfig = { + model: cfg.model, + reasoning: cfg.reasoning, + apiKey, // Direct API key + }; + + // Yield events from agentLoop + for await (const ev of agentLoop(userMessage as unknown as UserMessage, context, pc, signal)) { + yield ev; + } + } +} +``` + +**Verification**: Can send message directly to provider APIs + +#### Task 3.5: Create Transport Configuration +**File**: `src/utils/config.ts` (NEW) +**Actions**: +1. Create transport mode storage: "direct" | "proxy" +2. Create proxy URL storage with default +3. Create getters/setters: + - `getTransportMode()` / `setTransportMode()` + - `getProxyUrl()` / `setProxyUrl()` +4. Store in chrome.storage.local + +**Example**: +```typescript +export type TransportMode = "direct" | "proxy"; + +export async function getTransportMode(): Promise { + const result = await chrome.storage.local.get("transport-mode"); + return (result["transport-mode"] as TransportMode) || "direct"; +} + +export async function setTransportMode(mode: TransportMode): Promise { + await chrome.storage.local.set({ "transport-mode": mode }); +} + +export async function getProxyUrl(): Promise { + const result = await chrome.storage.local.get("proxy-url"); + return result["proxy-url"] || "https://genai.mariozechner.at"; +} + +export async function setProxyUrl(url: string): Promise { + await chrome.storage.local.set({ "proxy-url": url }); +} +``` + +**Verification**: Can read/write transport config + +--- + +### Phase 4: State Management + +#### Task 4.1: Port Utilities +**File**: `src/utils/format.ts` +**Actions**: +1. Read file to identify all formatting functions +2. Copy `formatUsage()` function +3. Copy any other utilities needed by AgentSession +4. Update imports + +**Verification**: Test formatUsage with sample usage data + +#### Task 4.2: Port Auth Token Utils +**File**: `src/utils/auth-token.ts` +**Dependencies**: PromptDialog +**Actions**: +1. Copy file to `src/utils/` +2. Update to use chrome.storage.local +3. Will need PromptDialog (next task) + +**Verification**: Can prompt for and store auth token + +#### Task 4.3: Port/Create PromptDialog +**File**: `src/dialogs/PromptDialog.ts` +**Actions**: +1. Read genai-workshop-new version +2. Adapt to use `@mariozechner/mini-lit` Dialog +3. Create simple input dialog similar to ApiKeysDialog +4. Add `PromptDialog.ask(title, message, defaultValue, isPassword)` static method + +**Verification**: Can prompt for text input + +#### Task 4.4: Port Agent Session +**File**: `src/state/agent-session.ts` +**Dependencies**: Transports, formatUsage, auth-token, DebugView types +**Actions**: +1. Copy `agent-session.ts` to `src/state/` +2. Update imports: + - ProxyTransport from `./transports/proxy-transport.js` + - DirectTransport from `./transports/direct-transport.js` + - Types from pi-ai + - KeyStore, auth-token utils +3. Modify constructor to accept transport mode +4. Create transport based on mode: + - "proxy" → ProxyTransport + - "direct" → DirectTransport +5. Update to use chrome.storage for persistence +6. Add `ThinkingLevel` type +7. Add `AppMessage` type extension for attachments + +**Key modifications**: +```typescript +constructor(opts: AgentSessionOptions & { transportMode?: TransportMode } = { + authTokenProvider: async () => getAuthToken() +}) { + // ... existing state init ... + + const mode = opts.transportMode || await getTransportMode(); + + if (mode === "proxy") { + this.transport = new ProxyTransport( + async () => this.preprocessMessages(), + opts.authTokenProvider + ); + } else { + this.transport = new DirectTransport( + async () => this.preprocessMessages() + ); + } +} +``` + +**Verification**: +- Create session with direct transport, send message +- Create session with proxy transport, send message +- Test abort functionality +- Test state subscription + +--- + +### Phase 5: Main Interface Integration + +#### Task 5.1: Port AgentInterface +**File**: `src/AgentInterface.ts` +**Dependencies**: Everything above +**Actions**: +1. Copy `AgentInterface.ts` to `src/` +2. Update all imports to use ported components +3. Register `agent-interface` custom element with guard +4. Update icons to lucide +5. Add i18n strings: + - "No session available", "No session set", "Hide debug view", "Show debug view" +6. Properties: + - `session` (external AgentSession) + - `enableAttachments` + - `enableModelSelector` + - `enableThinking` + - `showThemeToggle` + - `showDebugToggle` +7. Methods: + - `setInput(text, attachments)` + - `sendMessage(input, attachments)` + +**Verification**: Full chat interface works end-to-end + +#### Task 5.2: Integrate into ChatPanel +**File**: `src/ChatPanel.ts` +**Actions**: +1. Remove current chat implementation +2. Create AgentSession instance +3. Render `` with session +4. Configure: + - `enableAttachments={true}` + - `enableModelSelector={true}` + - `enableThinking={true}` + - `showThemeToggle={false}` (already in header) + - `showDebugToggle={false}` (optional) +5. Remove old MessageEditor integration (now inside AgentInterface) +6. Set system prompt (optional) +7. Set default tools (optional - calculateTool, getCurrentTimeTool) + +**Example**: +```typescript +import { AgentSession } from "./state/agent-session.js"; +import "./AgentInterface.js"; +import { calculateTool, getCurrentTimeTool } from "./tools/index.js"; + +@customElement("chat-panel") +export class ChatPanel extends LitElement { + @state() private session!: AgentSession; + + override async connectedCallback() { + super.connectedCallback(); + + // Create session + this.session = new AgentSession({ + initialState: { + systemPrompt: "You are a helpful AI assistant.", + tools: [calculateTool, getCurrentTimeTool], + }, + authTokenProvider: async () => getAuthToken(), + transportMode: await getTransportMode(), + }); + } + + override render() { + return html` + + `; + } +} +``` + +**Verification**: Full extension works with chat interface + +#### Task 5.3: Create Settings Dialog +**File**: `src/dialogs/SettingsDialog.ts` (NEW) +**Actions**: +1. Create dialog extending DialogBase +2. Sections: + - **Transport Mode**: Radio buttons for "Direct" | "Proxy" + - **Proxy URL**: Input field (only shown if proxy mode) + - **API Keys**: Button to open ApiKeysDialog +3. Save settings to config utils + +**UI Layout**: +``` +┌─────────────────────────────────────┐ +│ Settings [x] │ +├─────────────────────────────────────┤ +│ │ +│ Transport Mode │ +│ ○ Direct (use API keys) │ +│ ● Proxy (use auth token) │ +│ │ +│ Proxy URL │ +│ [https://genai.mariozechner.at ] │ +│ │ +│ [Manage API Keys...] │ +│ │ +│ [Cancel] [Save] │ +└─────────────────────────────────────┘ +``` + +**Verification**: Can toggle transport mode and set proxy URL + +#### Task 5.4: Update Header +**File**: `src/sidepanel.ts` +**Actions**: +1. Change settings button to open SettingsDialog (not ApiKeysDialog directly) +2. SettingsDialog should have button to open ApiKeysDialog + +**Verification**: Settings accessible from header + +--- + +### Phase 6: Optional Features + +#### Task 6.1: Port DebugView (Optional) +**File**: `src/DebugView.ts` +**Actions**: +1. Read full file to understand functionality +2. Copy to `src/` +3. Update imports +4. Format ChatML, SSE events, timing info +5. Add to AgentInterface when `showDebugToggle={true}` + +**Verification**: Debug view shows request/response details + +#### Task 6.2: Port Session Store (Optional) +**File**: `src/utils/session-db.ts` or `src/state/session-store.ts` +**Actions**: +1. Read file to understand IndexedDB usage +2. Create IndexedDB schema for sessions +3. Implement save/load/list/delete operations +4. Add to AgentInterface or ChatPanel +5. Add UI for switching sessions + +**Verification**: Can save and load conversation history + +#### Task 6.3: Add System Prompt Editor (Optional) +**Actions**: +1. Create dialog or expandable textarea +2. Allow editing session.state.systemPrompt +3. Add to settings or main interface + +**Verification**: Can customize system prompt + +--- + +## File Mapping Reference + +### Source → Destination + +| Source File | Destination File | Status | Dependencies | +|------------|------------------|--------|--------------| +| `app/ConsoleBlock.ts` | `src/ConsoleBlock.ts` | ⭕ New | mini-lit, lucide | +| `app/Messages.ts` | `src/Messages.ts` | ⭕ New | ConsoleBlock, formatUsage, tools | +| `app/MessageList.ts` | `src/MessageList.ts` | ⭕ New | Messages.ts | +| `app/StreamingMessageContainer.ts` | `src/StreamingMessageContainer.ts` | ⭕ New | Messages.ts | +| `app/AgentInterface.ts` | `src/AgentInterface.ts` | ⭕ New | All message components | +| `app/state/agent-session.ts` | `src/state/agent-session.ts` | ⭕ New | Transports, formatUsage | +| `app/state/transports/types.ts` | `src/state/transports/types.ts` | ⭕ New | pi-ai | +| `app/state/transports/proxy-transport.ts` | `src/state/transports/proxy-transport.ts` | ⭕ New | proxy-client | +| N/A | `src/state/transports/direct-transport.ts` | ⭕ New | pi-ai, KeyStore | +| `app/utils/proxy-client.ts` | `src/utils/proxy-client.ts` | ⭕ New | auth-token | +| N/A | `src/utils/config.ts` | ⭕ New | chrome.storage | +| `app/utils/format.ts` | `src/utils/format.ts` | ⭕ New | None | +| `app/utils/auth-token.ts` | `src/utils/auth-token.ts` | ⭕ New | PromptDialog | +| `app/tools/types.ts` | `src/tools/types.ts` | ⭕ New | None | +| `app/tools/renderer-registry.ts` | `src/tools/renderer-registry.ts` | ⭕ New | types.ts | +| `app/tools/renderers/DefaultRenderer.ts` | `src/tools/renderers/DefaultRenderer.ts` | ⭕ New | mini-lit | +| `app/tools/renderers/CalculateRenderer.ts` | `src/tools/renderers/CalculateRenderer.ts` | ⭕ New | mini-lit | +| `app/tools/renderers/GetCurrentTimeRenderer.ts` | `src/tools/renderers/GetCurrentTimeRenderer.ts` | ⭕ New | mini-lit | +| `app/tools/renderers/BashRenderer.ts` | `src/tools/renderers/BashRenderer.ts` | ⭕ New | ConsoleBlock | +| `app/tools/javascript-repl.ts` | `src/tools/javascript-repl.ts` | ⭕ New | pi-ai | +| `app/tools/web-search.ts` | `src/tools/web-search.ts` | ⭕ New | pi-ai | +| `app/tools/sleep.ts` | `src/tools/sleep.ts` | ⭕ New | pi-ai | +| `app/tools/index.ts` | `src/tools/index.ts` | ⭕ New | All tools | +| `app/dialogs/PromptDialog.ts` | `src/dialogs/PromptDialog.ts` | ⭕ New | mini-lit | +| N/A | `src/dialogs/SettingsDialog.ts` | ⭕ New | config, ApiKeysDialog | +| `app/DebugView.ts` | `src/DebugView.ts` | ⭕ Optional | highlight.js | +| `app/utils/session-db.ts` | `src/utils/session-db.ts` | ⭕ Optional | IndexedDB | + +### Already in Extension + +| File | Status | Notes | +|------|--------|-------| +| `src/MessageEditor.ts` | ✅ Exists | May need minor updates | +| `src/AttachmentTile.ts` | ✅ Exists | Complete | +| `src/AttachmentOverlay.ts` | ✅ Exists | Complete | +| `src/utils/attachment-utils.ts` | ✅ Exists | Complete | +| `src/dialogs/ModelSelector.ts` | ✅ Exists | May need integration check | +| `src/dialogs/ApiKeysDialog.ts` | ✅ Exists | Complete | +| `src/state/KeyStore.ts` | ✅ Exists | Complete | + +--- + +## Critical Implementation Notes + +### 1. Custom Element Registration Guards + +ALL custom elements must use registration guards to prevent duplicate registration errors: + +```typescript +// Instead of @customElement decorator +export class MyComponent extends LitElement { + // ... component code ... +} + +// At end of file +if (!customElements.get("my-component")) { + customElements.define("my-component", MyComponent); +} +``` + +### 2. Import Path Updates + +When porting, update ALL imports: + +**From genai-workshop-new**: +```typescript +import { Button } from "./mini/Button.js"; +import { iconLoader4Line } from "./mini/icons.js"; +``` + +**To browser extension**: +```typescript +import { Button } from "@mariozechner/mini-lit"; +import { Loader2 } from "lucide"; +import { icon } from "@mariozechner/mini-lit"; +// Use: icon(Loader2, "md") +``` + +### 3. Icon Mapping + +| genai-workshop | lucide | Usage | +|----------------|--------|-------| +| `iconLoader4Line` | `Loader2` | `icon(Loader2, "sm")` | +| `iconToolsLine` | `Wrench` | `icon(Wrench, "md")` | +| `iconBugLine` | `Bug` | `icon(Bug, "sm")` | +| `iconCheckLine` | `Check` | `icon(Check, "sm")` | +| `iconFileCopy2Line` | `Copy` | `icon(Copy, "sm")` | + +### 4. Chrome Extension APIs + +Replace browser APIs where needed: +- `localStorage` → `chrome.storage.local` +- `fetch("/api/...")` → `fetch(proxyUrl + "/api/...")` +- No direct filesystem access + +### 5. Transport Mode Configuration + +Ensure AgentSession can be created with either transport: + +```typescript +// Direct mode (uses API keys from KeyStore) +const session = new AgentSession({ + transportMode: "direct", + authTokenProvider: async () => undefined, // not needed +}); + +// Proxy mode (uses auth token) +const session = new AgentSession({ + transportMode: "proxy", + authTokenProvider: async () => getAuthToken(), +}); +``` + +### 6. i18n Strings to Add + +All UI strings must be in i18n.ts with English and German translations: + +```typescript +// Messages.ts +"Error:", "Request aborted", "Call", "Result", "(no result)", +"Waiting for tool result…", "Call was aborted; no result." + +// ConsoleBlock.ts +"console", "Copy output", "Copied!" + +// AgentInterface.ts +"No session available", "No session set", "Hide debug view", "Show debug view" + +// Transport errors +"Proxy error: {status} {statusText}", "Proxy error: {error}", +"Auth token is required for proxy transport" + +// Settings +"Settings", "Transport Mode", "Direct (use API keys)", +"Proxy (use auth token)", "Proxy URL", "Manage API Keys" +``` + +### 7. TypeScript Configuration + +The extension uses `useDefineForClassFields: false` in tsconfig.base.json. Ensure all ported components are compatible. + +### 8. Build Verification Steps + +After each phase: +1. Run `npm run check` - TypeScript compilation +2. Run `npm run build:chrome` - Chrome extension build +3. Run `npm run build:firefox` - Firefox extension build +4. Load extension in browser and test functionality +5. Check console for errors + +### 9. Proxy URL Configuration + +Default proxy URL should be configurable but default to: +```typescript +const DEFAULT_PROXY_URL = "https://genai.mariozechner.at"; +``` + +Users should be able to change this in settings for self-hosted proxies. + +--- + +## Testing Checklist + +### Phase 1: Message Rendering +- [ ] User messages display with text +- [ ] User messages display with attachments +- [ ] Assistant messages display with text +- [ ] Assistant messages display with thinking blocks +- [ ] Assistant messages display with tool calls +- [ ] Tool messages show pending state with spinner +- [ ] Tool messages show completed state with results +- [ ] Tool messages show error state +- [ ] Tool messages show aborted state +- [ ] Console blocks render output +- [ ] Console blocks auto-scroll +- [ ] Console blocks copy to clipboard + +### Phase 2: Tool System +- [ ] Calculate tool renders expression and result +- [ ] Time tool renders timezone and formatted time +- [ ] Bash tool renders output in console block +- [ ] JavaScript REPL tool executes code +- [ ] Web search tool fetches results +- [ ] Sleep tool delays execution +- [ ] Custom tool renderers can be registered +- [ ] Unknown tools use default renderer +- [ ] Tool debug view shows call args and results + +### Phase 3: Transport Layer +- [ ] Proxy transport connects to server +- [ ] Proxy transport handles auth token +- [ ] Proxy transport streams messages +- [ ] Proxy transport reconstructs partial messages +- [ ] Proxy transport handles abort +- [ ] Proxy transport handles errors +- [ ] Direct transport uses API keys from KeyStore +- [ ] Direct transport calls provider APIs directly +- [ ] Direct transport handles missing API key +- [ ] Direct transport streams messages +- [ ] Direct transport handles abort +- [ ] Transport mode can be switched +- [ ] Proxy URL can be configured + +### Phase 4: State Management +- [ ] AgentSession manages conversation state +- [ ] AgentSession sends messages +- [ ] AgentSession receives streaming updates +- [ ] AgentSession handles tool execution +- [ ] AgentSession handles errors +- [ ] AgentSession can be aborted +- [ ] AgentSession persists state +- [ ] AgentSession supports multiple sessions +- [ ] System prompt can be set +- [ ] Model can be selected +- [ ] Thinking level can be adjusted +- [ ] Tools can be configured +- [ ] Usage stats are tracked + +### Phase 5: Main Interface +- [ ] AgentInterface displays messages +- [ ] AgentInterface handles scrolling +- [ ] AgentInterface enables auto-scroll +- [ ] AgentInterface shows usage stats +- [ ] AgentInterface integrates MessageEditor +- [ ] AgentInterface integrates ModelSelector +- [ ] AgentInterface shows thinking toggle +- [ ] Settings dialog opens +- [ ] Settings dialog saves transport mode +- [ ] Settings dialog saves proxy URL +- [ ] Settings dialog opens API keys dialog +- [ ] Header settings button works + +### Phase 6: Optional Features +- [ ] Debug view shows request details +- [ ] Debug view shows response details +- [ ] Debug view shows timing info +- [ ] Debug view formats ChatML +- [ ] Sessions can be saved +- [ ] Sessions can be loaded +- [ ] Sessions can be listed +- [ ] Sessions can be deleted +- [ ] System prompt can be edited + +--- + +## Dependencies to Install + +```bash +# If not already installed +npm install highlight.js # For DebugView (optional) +``` + +--- + +## Estimated Complexity + +| Phase | Files | LOC (approx) | Complexity | Time Estimate | +|-------|-------|--------------|------------|---------------| +| Phase 1 | 5 | ~800 | Medium | 4-6 hours | +| Phase 2 | 10 | ~400 | Low-Medium | 3-4 hours | +| Phase 3 | 6 | ~600 | High | 6-8 hours | +| Phase 4 | 5 | ~500 | Medium-High | 5-7 hours | +| Phase 5 | 4 | ~400 | Medium | 4-5 hours | +| Phase 6 | 3 | ~400 | Low | 2-3 hours | +| **TOTAL** | **33** | **~3100** | - | **24-33 hours** | + +--- + +## Success Criteria + +The port is complete when: + +1. ✅ User can send messages with text and attachments +2. ✅ Messages stream in real-time with proper rendering +3. ✅ Tool calls execute and display results +4. ✅ Both direct and proxy transports work +5. ✅ Settings can be configured and persisted +6. ✅ Usage stats are tracked and displayed +7. ✅ Extension works in both Chrome and Firefox +8. ✅ All TypeScript types compile without errors +9. ✅ No console errors in normal operation +10. ✅ UI is responsive and performs well + +--- + +## Notes for New Session + +If starting a new session, key context: + +1. **Extension structure**: Browser extension in `packages/browser-extension/` +2. **Source codebase**: `genai-workshop-new/src/app/` +3. **UI framework**: LitElement with `@mariozechner/mini-lit` package +4. **AI package**: `@mariozechner/pi-ai` for LLM interactions +5. **Icons**: Using lucide instead of custom icon set +6. **i18n**: All UI strings must be in i18n.ts (English + German) +7. **Storage**: chrome.storage.local for all persistence +8. **TypeScript**: `useDefineForClassFields: false` required +9. **Custom elements**: Must use registration guards +10. **Build**: `npm run build:chrome` and `npm run build:firefox` + +**Critical files to reference**: +- `packages/browser-extension/tsconfig.json` - TS config +- `packages/browser-extension/src/utils/i18n.ts` - i18n strings +- `packages/browser-extension/src/state/KeyStore.ts` - API key storage +- `packages/browser-extension/src/dialogs/ApiKeysDialog.ts` - API key UI +- `genai-workshop-new/src/app/AgentInterface.ts` - Reference implementation +- `genai-workshop-new/src/app/state/agent-session.ts` - State management reference + +**Key architectural decisions**: +- Single AgentSession per chat +- Transport is pluggable (direct or proxy) +- Tools are registered in a global registry +- Message rendering is separated: stable (MessageList) vs streaming (StreamingMessageContainer) +- All components use light DOM (`createRenderRoot() { return this; }`) diff --git a/packages/browser-extension/README.md b/packages/browser-extension/README.md index 348bf4f0..e81d5ad8 100644 --- a/packages/browser-extension/README.md +++ b/packages/browser-extension/README.md @@ -10,12 +10,639 @@ A cross-browser extension that provides an AI-powered reading assistant in a sid ## Architecture -The extension adapts to each browser's UI paradigm: +### High-Level Overview + +The extension is a full-featured AI chat interface that runs in your browser's side panel/sidebar. It can communicate with AI providers in two ways: + +1. **Direct Mode** (default) - Calls AI provider APIs directly from the browser using API keys stored locally +2. **Proxy Mode** - Routes requests through a proxy server using an auth token + +**Browser Adaptation:** - **Chrome/Edge** - Side Panel API for dedicated panel UI - **Firefox** - Sidebar Action API for sidebar UI -- **Direct API Access** - Both can call AI APIs directly (no background worker needed) - **Page Content Access** - Uses `chrome.scripting.executeScript` to extract page text +### Core Architecture Layers + +``` +┌─────────────────────────────────────────────────────────────┐ +│ UI Layer (sidepanel.ts) │ +│ ├─ Header (theme toggle, settings) │ +│ └─ ChatPanel │ +│ └─ AgentInterface (main chat UI) │ +│ ├─ MessageList (stable messages) │ +│ ├─ StreamingMessageContainer (live updates) │ +│ └─ MessageEditor (input + attachments) │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ State Layer (state/) │ +│ └─ AgentSession │ +│ ├─ Manages conversation state │ +│ ├─ Coordinates transport │ +│ └─ Handles tool execution │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ Transport Layer (state/transports/) │ +│ ├─ DirectTransport (uses KeyStore for API keys) │ +│ └─ ProxyTransport (uses auth token + proxy server) │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ AI Provider APIs / Proxy Server │ +│ (Anthropic, OpenAI, Google, etc.) │ +└─────────────────────────────────────────────────────────────┘ +``` + +### Directory Structure by Responsibility + +``` +src/ +├── UI Components (what users see) +│ ├── sidepanel.ts # App entry point, header +│ ├── ChatPanel.ts # Main chat container, creates AgentSession +│ ├── AgentInterface.ts # Complete chat UI (messages + input) +│ ├── MessageList.ts # Renders stable messages +│ ├── StreamingMessageContainer.ts # Handles streaming updates +│ ├── Messages.ts # Message components (user, assistant, tool) +│ ├── MessageEditor.ts # Input field with attachments +│ ├── ConsoleBlock.ts # Console-style output display +│ ├── AttachmentTile.ts # Attachment preview thumbnails +│ ├── AttachmentOverlay.ts # Full-screen attachment viewer +│ └── ModeToggle.ts # Toggle between document/text view +│ +├── Dialogs (modal interactions) +│ ├── dialogs/ +│ │ ├── DialogBase.ts # Base class for all dialogs +│ │ ├── ModelSelector.ts # Select AI model +│ │ ├── ApiKeysDialog.ts # Manage API keys (for direct mode) +│ │ └── PromptDialog.ts # Simple text input dialog +│ +├── State Management (business logic) +│ ├── state/ +│ │ ├── agent-session.ts # Core state manager (pub/sub pattern) +│ │ ├── KeyStore.ts # API key storage (Chrome local storage) +│ │ └── transports/ +│ │ ├── types.ts # Transport interface definitions +│ │ ├── DirectTransport.ts # Direct API calls +│ │ └── ProxyTransport.ts # Proxy server calls +│ +├── Tools (AI function calling) +│ ├── tools/ +│ │ ├── types.ts # ToolRenderer interface +│ │ ├── renderer-registry.ts # Global tool renderer registry +│ │ ├── index.ts # Tool exports and registration +│ │ └── renderers/ # Custom tool UI renderers +│ │ ├── DefaultRenderer.ts # Fallback for unknown tools +│ │ ├── CalculateRenderer.ts # Calculator tool UI +│ │ ├── GetCurrentTimeRenderer.ts +│ │ └── BashRenderer.ts # Bash command execution UI +│ +├── Utilities (shared helpers) +│ └── utils/ +│ ├── attachment-utils.ts # PDF, Office, image processing +│ ├── auth-token.ts # Proxy auth token management +│ ├── format.ts # Token usage, cost formatting +│ └── i18n.ts # Internationalization (EN + DE) +│ +└── Entry Points (browser integration) + ├── background.ts # Service worker (opens side panel) + ├── sidepanel.html # HTML entry point + └── live-reload.ts # Hot reload during development +``` + +--- + +## Common Development Tasks + +### "I want to add a new AI tool" + +**Tools** are functions the AI can call (e.g., calculator, web search, code execution). Here's how to add one: + +#### 1. Define the Tool (use `@mariozechner/pi-ai`) + +Tools come from the `@mariozechner/pi-ai` package. Use existing tools or create custom ones: + +```typescript +// src/tools/my-custom-tool.ts +import type { AgentTool } from "@mariozechner/pi-ai"; + +export const myCustomTool: AgentTool = { + name: "my_custom_tool", + label: "My Custom Tool", + description: "Does something useful", + parameters: { + type: "object", + properties: { + input: { type: "string", description: "Input parameter" } + }, + required: ["input"] + }, + execute: async (params) => { + // Your tool logic here + const result = processInput(params.input); + return { + output: result, + details: { /* any structured data */ } + }; + } +}; +``` + +#### 2. Create a Custom Renderer (Optional) + +Renderers control how the tool appears in the chat. If you don't create one, `DefaultRenderer` will be used. + +```typescript +// src/tools/renderers/MyCustomRenderer.ts +import { html } from "@mariozechner/mini-lit"; +import type { ToolResultMessage } from "@mariozechner/pi-ai"; +import type { ToolRenderer } from "../types.js"; + +export class MyCustomRenderer implements ToolRenderer { + renderParams(params: any, isStreaming?: boolean) { + // Show tool call parameters (e.g., "Searching for: ") + return html` +
+ ${isStreaming ? "Processing..." : `Input: ${params.input}`} +
+ `; + } + + renderResult(params: any, result: ToolResultMessage) { + // Show tool result (e.g., search results, calculation output) + if (result.isError) { + return html`
${result.output}
`; + } + return html` +
+
Result:
+
${result.output}
+
+ `; + } +} +``` + +**Renderer Tips:** +- Use `ConsoleBlock` for command output (see `BashRenderer.ts`) +- Use `` for code/JSON (from `@mariozechner/mini-lit`) +- Use `` for markdown content +- Check `isStreaming` to show loading states + +#### 3. Register the Tool and Renderer + +```typescript +// src/tools/index.ts +import { myCustomTool } from "./my-custom-tool.js"; +import { MyCustomRenderer } from "./renderers/MyCustomRenderer.js"; +import { registerToolRenderer } from "./renderer-registry.js"; + +// Register the renderer +registerToolRenderer("my_custom_tool", new MyCustomRenderer()); + +// Export the tool so ChatPanel can use it +export { myCustomTool }; +``` + +#### 4. Add Tool to ChatPanel + +```typescript +// src/ChatPanel.ts +import { myCustomTool } from "./tools/index.js"; + +// In AgentSession constructor: +this.session = new AgentSession({ + initialState: { + tools: [calculateTool, getCurrentTimeTool, myCustomTool], // Add here + // ... + } +}); +``` + +**File Locations:** +- Tool definition: `src/tools/my-custom-tool.ts` +- Tool renderer: `src/tools/renderers/MyCustomRenderer.ts` +- Registration: `src/tools/index.ts` (register renderer) +- Integration: `src/ChatPanel.ts` (add to tools array) + +--- + +### "I want to change how messages are displayed" + +**Message components** control how conversations appear: + +- **User messages**: Edit `UserMessage` in `src/Messages.ts` +- **Assistant messages**: Edit `AssistantMessage` in `src/Messages.ts` +- **Tool call cards**: Edit `ToolMessage` in `src/Messages.ts` +- **Markdown rendering**: Comes from `@mariozechner/mini-lit` (can't customize easily) +- **Code blocks**: Comes from `@mariozechner/mini-lit` (can't customize easily) + +**Example: Change user message styling** + +```typescript +// src/Messages.ts - in UserMessage component +render() { + return html` +
+ + +
+ `; +} +``` + +--- + +### "I want to add a new model provider" + +Models come from `@mariozechner/pi-ai`. The package supports: +- `anthropic` (Claude) +- `openai` (GPT) +- `google` (Gemini) +- `groq`, `cerebras`, `xai`, `openrouter`, etc. + +**To add a provider:** + +1. Ensure `@mariozechner/pi-ai` supports it (check package docs) +2. Add API key configuration in `src/dialogs/ApiKeysDialog.ts`: + - Add provider to `PROVIDERS` array + - Add test model to `TEST_MODELS` object +3. Users can then select models via the model selector + +**No code changes needed** - the extension auto-discovers all models from `@mariozechner/pi-ai`. + +--- + +### "I want to modify the transport layer" + +**Transport** determines how requests reach AI providers: + +#### Direct Mode (Default) +- **File**: `src/state/transports/DirectTransport.ts` +- **How it works**: Gets API keys from `KeyStore` → calls provider APIs directly +- **When to use**: Local development, no proxy server +- **Configuration**: API keys stored in Chrome local storage + +#### Proxy Mode +- **File**: `src/state/transports/ProxyTransport.ts` +- **How it works**: Gets auth token → sends request to proxy server → proxy calls providers +- **When to use**: Want to hide API keys, centralized auth, usage tracking +- **Configuration**: Auth token stored in localStorage, proxy URL hardcoded + +**Switch transport mode in ChatPanel:** + +```typescript +// src/ChatPanel.ts +this.session = new AgentSession({ + transportMode: "direct", // or "proxy" + authTokenProvider: async () => getAuthToken(), // Only needed for proxy + // ... +}); +``` + +**Proxy Server Requirements:** +- Must accept POST to `/api/stream` endpoint +- Request format: `{ model, context, options }` +- Response format: SSE stream with delta events +- See `ProxyTransport.ts` for expected event types + +**To add a new transport:** + +1. Create `src/state/transports/MyTransport.ts` +2. Implement `AgentTransport` interface: + ```typescript + async *run(userMessage, cfg, signal): AsyncIterable + ``` +3. Register in `ChatPanel.ts` constructor + +--- + +### "I want to change the system prompt" + +**System prompts** guide the AI's behavior. Change in `ChatPanel.ts`: + +```typescript +// src/ChatPanel.ts +this.session = new AgentSession({ + initialState: { + systemPrompt: "You are a helpful AI assistant specialized in code review.", + // ... + } +}); +``` + +Or make it dynamic: + +```typescript +// Read from storage, settings dialog, etc. +const systemPrompt = await chrome.storage.local.get("system-prompt"); +``` + +--- + +### "I want to add attachment support for a new file type" + +**Attachment processing** happens in `src/utils/attachment-utils.ts`: + +1. **Add file type detection** in `loadAttachment()`: + ```typescript + if (mimeType === "application/my-format" || fileName.endsWith(".myext")) { + const { extractedText } = await processMyFormat(arrayBuffer, fileName); + return { id, type: "document", fileName, mimeType, content, extractedText }; + } + ``` + +2. **Add processor function**: + ```typescript + async function processMyFormat(buffer: ArrayBuffer, fileName: string) { + // Extract text from your format + const text = extractTextFromMyFormat(buffer); + return { extractedText: `\n${text}\n` }; + } + ``` + +3. **Update accepted types** in `MessageEditor.ts`: + ```typescript + acceptedTypes = "image/*,application/pdf,.myext,..."; + ``` + +4. **Optional: Add preview support** in `AttachmentOverlay.ts` + +**Supported formats:** +- Images: All image/* (preview support) +- PDF: Text extraction + thumbnail generation +- Office: DOCX, PPTX, XLSX (text extraction) +- Text: .txt, .md, .json, .xml, etc. + +--- + +### "I want to customize the UI theme" + +The extension uses the **Claude theme** from `@mariozechner/mini-lit`. Colors are defined via CSS variables: + +**Option 1: Override theme variables** +```css +/* src/app.css */ +@layer base { + :root { + --primary: 210 100% 50%; /* Custom blue */ + --radius: 0.5rem; + } +} +``` + +**Option 2: Use a different mini-lit theme** +```css +/* src/app.css */ +@import "@mariozechner/mini-lit/themes/default.css"; /* Instead of claude.css */ +``` + +**Available variables:** +- `--background`, `--foreground` - Base colors +- `--card`, `--card-foreground` - Card backgrounds +- `--primary`, `--primary-foreground` - Primary actions +- `--muted`, `--muted-foreground` - Secondary elements +- `--accent`, `--accent-foreground` - Hover states +- `--destructive` - Error/delete actions +- `--border`, `--input` - Border colors +- `--radius` - Border radius + +--- + +### "I want to add a new settings option" + +Settings currently managed via dialogs. To add persistent settings: + +#### 1. Create storage helpers + +```typescript +// src/utils/config.ts (create this file) +export async function getMySetting(): Promise { + const result = await chrome.storage.local.get("my-setting"); + return result["my-setting"] || "default-value"; +} + +export async function setMySetting(value: string): Promise { + await chrome.storage.local.set({ "my-setting": value }); +} +``` + +#### 2. Create or extend settings dialog + +```typescript +// src/dialogs/SettingsDialog.ts (create this file, similar to ApiKeysDialog) +// Add UI for your setting +// Call getMySetting() / setMySetting() on save +``` + +#### 3. Open from header + +```typescript +// src/sidepanel.ts - in settings button onClick +SettingsDialog.open(); +``` + +#### 4. Use in ChatPanel + +```typescript +// src/ChatPanel.ts +const mySetting = await getMySetting(); +this.session = new AgentSession({ + initialState: { /* use mySetting */ } +}); +``` + +--- + +### "I want to access the current page content" + +Page content extraction is in `sidepanel.ts`: + +```typescript +// Example: Get page text +const [tab] = await chrome.tabs.query({ active: true, currentWindow: true }); +const results = await chrome.scripting.executeScript({ + target: { tabId: tab.id }, + func: () => document.body.innerText, +}); +const pageText = results[0].result; +``` + +**To use in chat:** +1. Extract page content in `ChatPanel` +2. Add to system prompt or first user message +3. Or create a tool that reads page content + +**Permissions required:** +- `activeTab` - Access current tab +- `scripting` - Execute scripts in pages +- Already configured in `manifest.*.json` + +--- + +## Transport Modes Explained + +### Direct Mode (Default) + +**Flow:** +``` +Browser Extension + → KeyStore (get API key) + → DirectTransport + → Provider API (Anthropic/OpenAI/etc.) + → Stream response back +``` + +**Pros:** +- No external dependencies +- Lower latency (direct connection) +- Works offline for API key management +- Full control over requests + +**Cons:** +- API keys stored in browser (secure, but local) +- Each user needs their own API keys +- CORS restrictions (some providers may not work) +- Can't track usage centrally + +**Setup:** +1. Open extension → Settings → Manage API Keys +2. Add keys for desired providers (Anthropic, OpenAI, etc.) +3. Select model and start chatting + +**Files involved:** +- `src/state/transports/DirectTransport.ts` - Transport implementation +- `src/state/KeyStore.ts` - API key storage +- `src/dialogs/ApiKeysDialog.ts` - API key UI + +--- + +### Proxy Mode + +**Flow:** +``` +Browser Extension + → Auth Token (from localStorage) + → ProxyTransport + → Proxy Server (https://genai.mariozechner.at or custom) + → Provider API + → Stream response back through proxy +``` + +**Pros:** +- No API keys in browser +- Centralized auth/usage tracking +- Can implement rate limiting, quotas +- Custom logic server-side +- No CORS issues + +**Cons:** +- Requires proxy server setup +- Additional network hop (latency) +- Dependency on proxy availability +- Need to manage auth tokens + +**Setup:** +1. Get auth token from proxy server admin +2. Extension prompts for token on first use +3. Token stored in localStorage +4. Start chatting (proxy handles provider APIs) + +**Proxy URL Configuration:** +Currently hardcoded in `ProxyTransport.ts`: +```typescript +const PROXY_URL = "https://genai.mariozechner.at"; +``` + +To make configurable: +1. Add storage helper in `utils/config.ts` +2. Add UI in SettingsDialog +3. Pass to ProxyTransport constructor + +**Proxy Server Requirements:** + +The proxy server must implement: + +**Endpoint:** `POST /api/stream` + +**Request:** +```typescript +{ + model: Model, // Provider + model ID + context: Context, // System prompt, messages, tools + options: { + temperature?: number, + maxTokens?: number, + reasoning?: string + } +} +``` + +**Response:** SSE (Server-Sent Events) stream + +**Event Types:** +```typescript +data: {"type":"start","partial":{...}} +data: {"type":"text_start","contentIndex":0} +data: {"type":"text_delta","contentIndex":0,"delta":"Hello"} +data: {"type":"text_end","contentIndex":0,"contentSignature":"..."} +data: {"type":"thinking_start","contentIndex":1} +data: {"type":"thinking_delta","contentIndex":1,"delta":"..."} +data: {"type":"toolcall_start","contentIndex":2,"id":"...","toolName":"..."} +data: {"type":"toolcall_delta","contentIndex":2,"delta":"..."} +data: {"type":"toolcall_end","contentIndex":2} +data: {"type":"done","reason":"stop","usage":{...}} +``` + +**Auth:** Bearer token in `Authorization` header + +**Error Handling:** +- Return 401 for invalid auth → extension clears token and re-prompts +- Return 4xx/5xx with JSON: `{"error":"message"}` + +**Reference Implementation:** +See `src/state/transports/ProxyTransport.ts` for full event parsing logic. + +--- + +### Switching Between Modes + +**At runtime** (in ChatPanel): +```typescript +const mode = await getTransportMode(); // "direct" or "proxy" +this.session = new AgentSession({ + transportMode: mode, + authTokenProvider: mode === "proxy" ? async () => getAuthToken() : undefined, + // ... +}); +``` + +**Storage helpers** (create these): +```typescript +// src/utils/config.ts +export type TransportMode = "direct" | "proxy"; + +export async function getTransportMode(): Promise { + const result = await chrome.storage.local.get("transport-mode"); + return (result["transport-mode"] as TransportMode) || "direct"; +} + +export async function setTransportMode(mode: TransportMode): Promise { + await chrome.storage.local.set({ "transport-mode": mode }); +} +``` + +**UI for switching** (create this): +```typescript +// src/dialogs/SettingsDialog.ts +// Radio buttons: ○ Direct (use API keys) / ○ Proxy (use auth token) +// On save: setTransportMode(), reload AgentSession +``` + +--- + ## Understanding mini-lit Before working on the UI, read these files to understand the component library: diff --git a/packages/browser-extension/src/AgentInterface.ts b/packages/browser-extension/src/AgentInterface.ts new file mode 100644 index 00000000..0c86e3bd --- /dev/null +++ b/packages/browser-extension/src/AgentInterface.ts @@ -0,0 +1,312 @@ +import { html, icon } from "@mariozechner/mini-lit"; +import type { ToolResultMessage, Usage } from "@mariozechner/pi-ai"; +import { LitElement } from "lit"; +import { customElement, property, query } from "lit/decorators.js"; +import { ApiKeysDialog } from "./dialogs/ApiKeysDialog.js"; +import { ModelSelector } from "./dialogs/ModelSelector.js"; +import type { MessageEditor } from "./MessageEditor.js"; +import "./MessageEditor.js"; +import "./MessageList.js"; +import "./Messages.js"; // Import for side effects to register the custom elements +import type { AgentSession, AgentSessionEvent } from "./state/agent-session.js"; +import { keyStore } from "./state/KeyStore.js"; +import "./StreamingMessageContainer.js"; +import type { StreamingMessageContainer } from "./StreamingMessageContainer.js"; +import type { Attachment } from "./utils/attachment-utils.js"; +import { formatUsage } from "./utils/format.js"; +import { i18n } from "./utils/i18n.js"; + +@customElement("agent-interface") +export class AgentInterface extends LitElement { + // Optional external session: when provided, this component becomes a view over the session + @property({ attribute: false }) session?: AgentSession; + @property() enableAttachments = true; + @property() enableModelSelector = true; + @property() enableThinking = true; + @property() showThemeToggle = false; + @property() showDebugToggle = false; + + // References + @query("message-editor") private _messageEditor!: MessageEditor; + @query("streaming-message-container") private _streamingContainer!: StreamingMessageContainer; + + private _autoScroll = true; + private _lastScrollTop = 0; + private _lastClientHeight = 0; + private _scrollContainer?: HTMLElement; + private _resizeObserver?: ResizeObserver; + private _unsubscribeSession?: () => void; + + public setInput(text: string, attachments?: Attachment[]) { + const update = () => { + if (!this._messageEditor) requestAnimationFrame(update); + else { + this._messageEditor.value = text; + this._messageEditor.attachments = attachments || []; + } + }; + update(); + } + + protected override createRenderRoot(): HTMLElement | DocumentFragment { + return this; + } + + override async connectedCallback() { + super.connectedCallback(); + + this.style.display = "flex"; + this.style.flexDirection = "column"; + this.style.height = "100%"; + this.style.minHeight = "0"; + + // Wait for first render to get scroll container + await this.updateComplete; + this._scrollContainer = this.querySelector(".overflow-y-auto") as HTMLElement; + + if (this._scrollContainer) { + // Set up ResizeObserver to detect content changes + this._resizeObserver = new ResizeObserver(() => { + if (this._autoScroll && this._scrollContainer) { + this._scrollContainer.scrollTop = this._scrollContainer.scrollHeight; + } + }); + + // Observe the content container inside the scroll container + const contentContainer = this._scrollContainer.querySelector(".max-w-3xl"); + if (contentContainer) { + this._resizeObserver.observe(contentContainer); + } + + // Set up scroll listener with better detection + this._scrollContainer.addEventListener("scroll", this._handleScroll); + } + + // Subscribe to external session if provided + this.setupSessionSubscription(); + + // Attach debug listener if session provided + if (this.session) { + this.session = this.session; // explicitly set to trigger subscription + } + } + + override disconnectedCallback() { + super.disconnectedCallback(); + + // Clean up observers and listeners + if (this._resizeObserver) { + this._resizeObserver.disconnect(); + this._resizeObserver = undefined; + } + + if (this._scrollContainer) { + this._scrollContainer.removeEventListener("scroll", this._handleScroll); + } + + if (this._unsubscribeSession) { + this._unsubscribeSession(); + this._unsubscribeSession = undefined; + } + } + + private setupSessionSubscription() { + if (this._unsubscribeSession) { + this._unsubscribeSession(); + this._unsubscribeSession = undefined; + } + if (!this.session) return; + this._unsubscribeSession = this.session.subscribe(async (ev: AgentSessionEvent) => { + if (ev.type === "state-update") { + if (this._streamingContainer) { + this._streamingContainer.isStreaming = ev.state.isStreaming; + this._streamingContainer.setMessage(ev.state.streamMessage, !ev.state.isStreaming); + } + this.requestUpdate(); + } else if (ev.type === "error-no-model") { + // TODO show some UI feedback + } else if (ev.type === "error-no-api-key") { + // Open API keys dialog to configure the missing key + ApiKeysDialog.open(); + } + }); + } + + private _handleScroll = (_ev: any) => { + if (!this._scrollContainer) return; + + const currentScrollTop = this._scrollContainer.scrollTop; + const scrollHeight = this._scrollContainer.scrollHeight; + const clientHeight = this._scrollContainer.clientHeight; + const distanceFromBottom = scrollHeight - currentScrollTop - clientHeight; + + // Ignore relayout due to message editor getting pushed up by stats + if (clientHeight < this._lastClientHeight) { + this._lastClientHeight = clientHeight; + return; + } + + // Only disable auto-scroll if user scrolled UP or is far from bottom + if (currentScrollTop !== 0 && currentScrollTop < this._lastScrollTop && distanceFromBottom > 50) { + this._autoScroll = false; + } else if (distanceFromBottom < 10) { + // Re-enable if very close to bottom + this._autoScroll = true; + } + + this._lastScrollTop = currentScrollTop; + this._lastClientHeight = clientHeight; + }; + + public async sendMessage(input: string, attachments?: Attachment[]) { + if ((!input.trim() && attachments?.length === 0) || this.session?.state.isStreaming) return; + const session = this.session; + if (!session) throw new Error("No session set on AgentInterface"); + if (!session.state.model) throw new Error("No model set on AgentInterface"); + + // Check if API key exists for the provider (only needed in direct mode) + const provider = session.state.model.provider; + let apiKey = await keyStore.getKey(provider); + + // If no API key, open the API keys dialog + if (!apiKey) { + await ApiKeysDialog.open(); + // Check again after dialog closes + apiKey = await keyStore.getKey(provider); + // If still no API key, abort the send + if (!apiKey) { + return; + } + } + + // Only clear editor after we know we can send + this._messageEditor.value = ""; + this._messageEditor.attachments = []; + this._autoScroll = true; // Enable auto-scroll when sending a message + + await this.session?.prompt(input, attachments); + } + + private renderMessages() { + if (!this.session) + return html`
${i18n("No session available")}
`; + const state = this.session.state; + // Build a map of tool results to allow inline rendering in assistant messages + const toolResultsById = new Map>(); + for (const message of state.messages) { + if (message.role === "toolResult") { + toolResultsById.set(message.toolCallId, message); + } + } + return html` +
+ + ()} + .isStreaming=${state.isStreaming} + > + + + +
+ `; + } + + private renderStats() { + if (!this.session) return html`
`; + + const state = this.session.state; + const totals = state.messages + .filter((m) => m.role === "assistant") + .reduce( + (acc, msg: any) => { + const usage = msg.usage; + if (usage) { + acc.input += usage.input; + acc.output += usage.output; + acc.cacheRead += usage.cacheRead; + acc.cacheWrite += usage.cacheWrite; + acc.cost.total += usage.cost.total; + } + return acc; + }, + { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + } satisfies Usage, + ); + + const hasTotals = totals.input || totals.output || totals.cacheRead || totals.cacheWrite; + const totalsText = hasTotals ? formatUsage(totals) : ""; + + return html` +
+
+ ${this.showThemeToggle ? html`` : html``} +
+
${totalsText ? html`${totalsText}` : ""}
+
+ `; + } + + override render() { + if (!this.session) + return html`
${i18n("No session set")}
`; + + const session = this.session; + const state = this.session.state; + return html` +
+ +
+
${this.renderMessages()}
+
+ + +
+
+ { + this.sendMessage(input, attachments); + }} + .onAbort=${() => session.abort()} + .onModelSelect=${() => { + ModelSelector.open(state.model, (model) => session.setModel(model)); + }} + .onThinkingChange=${ + this.enableThinking + ? (level: "off" | "minimal" | "low" | "medium" | "high") => { + session.setThinkingLevel(level); + } + : undefined + } + > + ${this.renderStats()} +
+
+
+ `; + } +} + +// Register custom element with guard +if (!customElements.get("agent-interface")) { + customElements.define("agent-interface", AgentInterface); +} diff --git a/packages/browser-extension/src/ChatPanel.ts b/packages/browser-extension/src/ChatPanel.ts index c985ceaf..730a9c1d 100644 --- a/packages/browser-extension/src/ChatPanel.ts +++ b/packages/browser-extension/src/ChatPanel.ts @@ -1,17 +1,14 @@ import { html } from "@mariozechner/mini-lit"; -import type { Model } from "@mariozechner/pi-ai"; -import { getModel } from "@mariozechner/pi-ai"; +import { calculateTool, getCurrentTimeTool, getModel } from "@mariozechner/pi-ai"; import { LitElement } from "lit"; import { customElement, state } from "lit/decorators.js"; -import { ModelSelector } from "./dialogs/ModelSelector.js"; -import "./MessageEditor.js"; -import type { Attachment } from "./utils/attachment-utils.js"; +import "./AgentInterface.js"; +import { AgentSession } from "./state/agent-session.js"; +import { getAuthToken } from "./utils/auth-token.js"; @customElement("pi-chat-panel") export class ChatPanel extends LitElement { - @state() currentModel: Model | null = null; - @state() messageText = ""; - @state() attachments: Attachment[] = []; + @state() private session!: AgentSession; createRenderRoot() { return this; @@ -19,50 +16,42 @@ export class ChatPanel extends LitElement { override async connectedCallback() { super.connectedCallback(); - // Set default model - this.currentModel = getModel("anthropic", "claude-3-5-haiku-20241022"); + + // Ensure panel fills height and allows flex layout + this.style.display = "flex"; + this.style.flexDirection = "column"; + this.style.height = "100%"; + this.style.minHeight = "0"; + + // Create agent session with default settings + this.session = new AgentSession({ + initialState: { + systemPrompt: "You are a helpful AI assistant.", + model: getModel("anthropic", "claude-3-5-haiku-20241022"), + tools: [calculateTool, getCurrentTimeTool], + thinkingLevel: "off", + }, + authTokenProvider: async () => getAuthToken(), + transportMode: "direct", // Use direct mode by default (API keys from KeyStore) + }); } - private handleSend = (text: string, attachments: Attachment[]) => { - // For now just alert and clear - alert(`Message: ${text}\nAttachments: ${attachments.length}`); - this.messageText = ""; - this.attachments = []; - }; - - private handleModelSelect = () => { - ModelSelector.open(this.currentModel, (model) => { - this.currentModel = model; - }); - }; - render() { - return html` -
- -
- -
+ if (!this.session) { + return html`
+
Loading...
+
`; + } - -
- { - this.messageText = value; - }} - .onSend=${this.handleSend} - .onModelSelect=${this.handleModelSelect} - .onFilesChange=${(files: Attachment[]) => { - this.attachments = files; - }} - > -
-
+ return html` + `; } } diff --git a/packages/browser-extension/src/ConsoleBlock.ts b/packages/browser-extension/src/ConsoleBlock.ts new file mode 100644 index 00000000..fc68d43f --- /dev/null +++ b/packages/browser-extension/src/ConsoleBlock.ts @@ -0,0 +1,67 @@ +import { html, icon } from "@mariozechner/mini-lit"; +import { LitElement } from "lit"; +import { property, state } from "lit/decorators.js"; +import { Check, Copy } from "lucide"; +import { i18n } from "./utils/i18n.js"; + +export class ConsoleBlock extends LitElement { + @property() content: string = ""; + @state() private copied = false; + + protected override createRenderRoot(): HTMLElement | DocumentFragment { + return this; + } + + override connectedCallback(): void { + super.connectedCallback(); + this.style.display = "block"; + } + + private async copy() { + try { + await navigator.clipboard.writeText(this.content || ""); + this.copied = true; + setTimeout(() => { + this.copied = false; + }, 1500); + } catch (e) { + console.error("Copy failed", e); + } + } + + override updated() { + // Auto-scroll to bottom on content changes + const container = this.querySelector(".console-scroll") as HTMLElement | null; + if (container) { + container.scrollTop = container.scrollHeight; + } + } + + override render() { + return html` +
+
+ ${i18n("console")} + +
+
+ 
+${this.content || ""}
+
+
+ `; + } +} + +// Register custom element +if (!customElements.get("console-block")) { + customElements.define("console-block", ConsoleBlock); +} diff --git a/packages/browser-extension/src/MessageList.ts b/packages/browser-extension/src/MessageList.ts new file mode 100644 index 00000000..7c6dc790 --- /dev/null +++ b/packages/browser-extension/src/MessageList.ts @@ -0,0 +1,82 @@ +import { html } from "@mariozechner/mini-lit"; +import type { + AgentTool, + AssistantMessage as AssistantMessageType, + Message, + ToolResultMessage as ToolResultMessageType, +} from "@mariozechner/pi-ai"; +import { LitElement, type TemplateResult } from "lit"; +import { property } from "lit/decorators.js"; +import { repeat } from "lit/directives/repeat.js"; + +export class MessageList extends LitElement { + @property({ type: Array }) messages: Message[] = []; + @property({ type: Array }) tools: AgentTool[] = []; + @property({ type: Object }) pendingToolCalls?: Set; + @property({ type: Boolean }) isStreaming: boolean = false; + + protected override createRenderRoot(): HTMLElement | DocumentFragment { + return this; + } + + override connectedCallback(): void { + super.connectedCallback(); + this.style.display = "block"; + } + + private buildRenderItems() { + // Map tool results by call id for quick lookup + const resultByCallId = new Map(); + for (const message of this.messages) { + if (message.role === "toolResult") { + resultByCallId.set(message.toolCallId, message); + } + } + + const items: Array<{ key: string; template: TemplateResult }> = []; + let index = 0; + for (const msg of this.messages) { + if (msg.role === "user") { + items.push({ + key: `msg:${index}`, + template: html``, + }); + index++; + } else if (msg.role === "assistant") { + const amsg = msg as AssistantMessageType; + items.push({ + key: `msg:${index}`, + template: html``, + }); + index++; + } else { + // Skip standalone toolResult messages; they are rendered via paired tool-message above + // For completeness, other roles are not expected + } + } + return items; + } + + override render() { + const items = this.buildRenderItems(); + return html`
+ ${repeat( + items, + (it) => it.key, + (it) => it.template, + )} +
`; + } +} + +// Register custom element +if (!customElements.get("message-list")) { + customElements.define("message-list", MessageList); +} diff --git a/packages/browser-extension/src/Messages.ts b/packages/browser-extension/src/Messages.ts new file mode 100644 index 00000000..b1524361 --- /dev/null +++ b/packages/browser-extension/src/Messages.ts @@ -0,0 +1,310 @@ +import { Button, html, icon } from "@mariozechner/mini-lit"; +import type { + AgentTool, + AssistantMessage as AssistantMessageType, + ToolCall, + ToolResultMessage as ToolResultMessageType, + UserMessage as UserMessageType, +} from "@mariozechner/pi-ai"; +import type { AgentToolResult } from "@mariozechner/pi-ai/dist/agent/types.js"; +import { LitElement, type TemplateResult } from "lit"; +import { customElement, property, state } from "lit/decorators.js"; +import { Bug, Loader, Wrench } from "lucide"; +import { renderToolParams, renderToolResult } from "./tools/index.js"; +import type { Attachment } from "./utils/attachment-utils.js"; +import { formatUsage } from "./utils/format.js"; +import { i18n } from "./utils/i18n.js"; + +export type UserMessageWithAttachments = UserMessageType & { attachments?: Attachment[] }; +export type AppMessage = AssistantMessageType | UserMessageWithAttachments | ToolResultMessageType; + +@customElement("user-message") +export class UserMessage extends LitElement { + @property({ type: Object }) message!: UserMessageWithAttachments; + + protected override createRenderRoot(): HTMLElement | DocumentFragment { + return this; + } + + override connectedCallback(): void { + super.connectedCallback(); + this.style.display = "block"; + } + + override render() { + const content = + typeof this.message.content === "string" + ? this.message.content + : this.message.content.find((c) => c.type === "text")?.text || ""; + + return html` +
+ + ${ + this.message.attachments && this.message.attachments.length > 0 + ? html` +
+ ${this.message.attachments.map( + (attachment) => html` `, + )} +
+ ` + : "" + } +
+ `; + } +} + +@customElement("assistant-message") +export class AssistantMessage extends LitElement { + @property({ type: Object }) message!: AssistantMessageType; + @property({ type: Array }) tools?: AgentTool[]; + @property({ type: Object }) pendingToolCalls?: Set; + @property({ type: Boolean }) hideToolCalls = false; + @property({ type: Object }) toolResultsById?: Map; + @property({ type: Boolean }) isStreaming: boolean = false; + + protected override createRenderRoot(): HTMLElement | DocumentFragment { + return this; + } + + override connectedCallback(): void { + super.connectedCallback(); + this.style.display = "block"; + } + + override render() { + // Render content in the order it appears + const orderedParts: TemplateResult[] = []; + + for (const chunk of this.message.content) { + if (chunk.type === "text" && chunk.text.trim() !== "") { + orderedParts.push(html``); + } else if (chunk.type === "thinking" && chunk.thinking.trim() !== "") { + orderedParts.push(html` `); + } else if (chunk.type === "toolCall") { + if (!this.hideToolCalls) { + const tool = this.tools?.find((t) => t.name === chunk.name); + const pending = this.pendingToolCalls?.has(chunk.id) ?? false; + const result = this.toolResultsById?.get(chunk.id); + const aborted = !pending && !result && !this.isStreaming; + orderedParts.push( + html``, + ); + } + } + } + + return html` +
+ ${orderedParts.length ? html`
${orderedParts}
` : ""} + ${ + this.message.usage + ? html`
${formatUsage(this.message.usage)}
` + : "" + } + ${ + this.message.stopReason === "error" && this.message.errorMessage + ? html` +
+ ${i18n("Error:")} ${this.message.errorMessage} +
+ ` + : "" + } + ${ + this.message.stopReason === "aborted" + ? html`${i18n("Request aborted")}` + : "" + } +
+ `; + } +} + +@customElement("tool-message-debug") +export class ToolMessageDebugView extends LitElement { + @property({ type: Object }) callArgs: any; + @property({ type: String }) result?: AgentToolResult; + @property({ type: Boolean }) hasResult: boolean = false; + + protected override createRenderRoot(): HTMLElement | DocumentFragment { + return this; // light DOM for shared styles + } + + override connectedCallback(): void { + super.connectedCallback(); + this.style.display = "block"; + } + + private pretty(value: unknown): { content: string; isJson: boolean } { + try { + if (typeof value === "string") { + const maybeJson = JSON.parse(value); + return { content: JSON.stringify(maybeJson, null, 2), isJson: true }; + } + return { content: JSON.stringify(value, null, 2), isJson: true }; + } catch { + return { content: typeof value === "string" ? value : String(value), isJson: false }; + } + } + + override render() { + const output = this.pretty(this.result?.output); + const details = this.pretty(this.result?.details); + + return html` +
+
+
${i18n("Call")}
+ +
+
+
${i18n("Result")}
+ ${ + this.hasResult + ? html` + ` + : html`
${i18n("(no result)")}
` + } +
+
+ `; + } +} + +@customElement("tool-message") +export class ToolMessage extends LitElement { + @property({ type: Object }) toolCall!: ToolCall; + @property({ type: Object }) tool?: AgentTool; + @property({ type: Object }) result?: ToolResultMessageType; + @property({ type: Boolean }) pending: boolean = false; + @property({ type: Boolean }) aborted: boolean = false; + @property({ type: Boolean }) isStreaming: boolean = false; + @state() private _showDebug = false; + + protected override createRenderRoot(): HTMLElement | DocumentFragment { + return this; + } + + override connectedCallback(): void { + super.connectedCallback(); + this.style.display = "block"; + } + + private toggleDebug = () => { + this._showDebug = !this._showDebug; + }; + + override render() { + const toolLabel = this.tool?.label || this.toolCall.name; + const toolName = this.tool?.name || this.toolCall.name; + const isError = this.result?.isError === true; + const hasResult = !!this.result; + + let statusIcon: TemplateResult; + if (this.pending || (this.isStreaming && !hasResult)) { + statusIcon = html`${icon(Loader, "md")}`; + } else if (this.aborted && !hasResult) { + statusIcon = html`${icon(Wrench, "md")}`; + } else if (hasResult && isError) { + statusIcon = html`${icon(Wrench, "md")}`; + } else if (hasResult) { + statusIcon = html`${icon(Wrench, "md")}`; + } else { + statusIcon = html`${icon(Wrench, "md")}`; + } + + // Normalize error text + let errorMessage = this.result?.output || ""; + if (isError) { + try { + const parsed = JSON.parse(errorMessage); + if ((parsed as any).error) errorMessage = (parsed as any).error; + else if ((parsed as any).message) errorMessage = (parsed as any).message; + } catch {} + errorMessage = errorMessage.replace(/^(Tool )?Error:\s*/i, ""); + errorMessage = errorMessage.replace(/^Error:\s*/i, ""); + } + + const paramsTpl = renderToolParams( + toolName, + this.toolCall.arguments, + this.isStreaming || (this.pending && !hasResult), + ); + const resultTpl = + hasResult && !isError ? renderToolResult(toolName, this.toolCall.arguments, this.result!) : undefined; + + return html` +
+
+
+ ${statusIcon} + ${toolLabel} +
+ ${Button({ + variant: this._showDebug ? "default" : "ghost", + size: "sm", + onClick: this.toggleDebug, + children: icon(Bug, "sm"), + className: "h-8 w-8", + })} +
+ + ${ + this._showDebug + ? html`` + : html` +
${paramsTpl}
+ ${ + this.pending && !hasResult + ? html`
${i18n("Waiting for tool result…")}
` + : "" + } + ${ + this.aborted && !hasResult + ? html`
${i18n("Call was aborted; no result.")}
` + : "" + } + ${ + hasResult && isError + ? html`
+ ${errorMessage} +
` + : "" + } + ${resultTpl ? html`
${resultTpl}
` : ""} + ` + } +
+ `; + } +} + +@customElement("aborted-message") +export class AbortedMessage extends LitElement { + protected override createRenderRoot(): HTMLElement | DocumentFragment { + return this; + } + + override connectedCallback(): void { + super.connectedCallback(); + this.style.display = "block"; + } + + protected override render(): unknown { + return html`${i18n("Request aborted")}`; + } +} diff --git a/packages/browser-extension/src/StreamingMessageContainer.ts b/packages/browser-extension/src/StreamingMessageContainer.ts new file mode 100644 index 00000000..af0d8e3c --- /dev/null +++ b/packages/browser-extension/src/StreamingMessageContainer.ts @@ -0,0 +1,99 @@ +import { html } from "@mariozechner/mini-lit"; +import type { AgentTool, Message, ToolResultMessage } from "@mariozechner/pi-ai"; +import { LitElement } from "lit"; +import { property, state } from "lit/decorators.js"; + +export class StreamingMessageContainer extends LitElement { + @property({ type: Array }) tools: AgentTool[] = []; + @property({ type: Boolean }) isStreaming = false; + @property({ type: Object }) pendingToolCalls?: Set; + @property({ type: Object }) toolResultsById?: Map; + + @state() private _message: Message | null = null; + private _pendingMessage: Message | null = null; + private _updateScheduled = false; + private _immediateUpdate = false; + + protected override createRenderRoot(): HTMLElement | DocumentFragment { + return this; + } + + override connectedCallback(): void { + super.connectedCallback(); + this.style.display = "block"; + } + + // Public method to update the message with batching for performance + public setMessage(message: Message | null, immediate = false) { + // Store the latest message + this._pendingMessage = message; + + // If this is an immediate update (like clearing), apply it right away + if (immediate || message === null) { + this._immediateUpdate = true; + this._message = message; + this.requestUpdate(); + // Cancel any pending updates since we're clearing + this._pendingMessage = null; + this._updateScheduled = false; + return; + } + + // Otherwise batch updates for performance during streaming + if (!this._updateScheduled) { + this._updateScheduled = true; + + requestAnimationFrame(async () => { + // Only apply the update if we haven't been cleared + if (!this._immediateUpdate && this._pendingMessage !== null) { + this._message = this._pendingMessage; + this.requestUpdate(); + } + // Reset for next batch + this._pendingMessage = null; + this._updateScheduled = false; + this._immediateUpdate = false; + }); + } + } + + override render() { + // Show loading indicator if loading but no message yet + if (!this._message) { + if (this.isStreaming) + return html`
+ +
`; + return html``; // Empty until a message is set + } + const msg = this._message; + + if (msg.role === "toolResult") { + // Skip standalone tool result in streaming; the stable list will render paired tool-message + return html``; + } else if (msg.role === "user") { + // Skip standalone tool result in streaming; the stable list will render it immediiately + return html``; + } else if (msg.role === "assistant") { + // Assistant message - render inline tool messages during streaming + return html` +
+ + ${this.isStreaming ? html`` : ""} +
+ `; + } + } +} + +// Register custom element +if (!customElements.get("streaming-message-container")) { + customElements.define("streaming-message-container", StreamingMessageContainer); +} diff --git a/packages/browser-extension/src/dialogs/ApiKeysDialog.ts b/packages/browser-extension/src/dialogs/ApiKeysDialog.ts index a54eec4e..d20d2a39 100644 --- a/packages/browser-extension/src/dialogs/ApiKeysDialog.ts +++ b/packages/browser-extension/src/dialogs/ApiKeysDialog.ts @@ -190,7 +190,7 @@ export class ApiKeysDialog extends DialogBase { (provider) => html`
- ${provider} + ${provider} ${ this.apiKeys[provider] ? Badge({ children: i18n("Configured"), variant: "default" }) diff --git a/packages/browser-extension/src/dialogs/PromptDialog.ts b/packages/browser-extension/src/dialogs/PromptDialog.ts index 86bd8ddb..849ee193 100644 --- a/packages/browser-extension/src/dialogs/PromptDialog.ts +++ b/packages/browser-extension/src/dialogs/PromptDialog.ts @@ -17,13 +17,18 @@ export class PromptDialog extends DialogBase { @property() isPassword = false; @state() private inputValue = ""; - private resolvePromise?: (value: string | null) => void; + private resolvePromise?: (value: string | undefined) => void; private inputRef = createRef(); protected override modalWidth = "min(400px, 90vw)"; protected override modalHeight = "auto"; - static async ask(title: string, message: string, defaultValue = "", isPassword = false): Promise { + static async ask( + title: string, + message: string, + defaultValue = "", + isPassword = false, + ): Promise { const dialog = new PromptDialog(); dialog.headerTitle = title; dialog.message = message; @@ -48,7 +53,7 @@ export class PromptDialog extends DialogBase { } private handleCancel() { - this.resolvePromise?.(null); + this.resolvePromise?.(undefined); this.close(); } diff --git a/packages/browser-extension/src/sidepanel.html b/packages/browser-extension/src/sidepanel.html index 6a370d99..b2768979 100644 --- a/packages/browser-extension/src/sidepanel.html +++ b/packages/browser-extension/src/sidepanel.html @@ -1,11 +1,11 @@ - + pi-ai - + \ No newline at end of file diff --git a/packages/browser-extension/src/sidepanel.ts b/packages/browser-extension/src/sidepanel.ts index 7b81503c..57575df4 100644 --- a/packages/browser-extension/src/sidepanel.ts +++ b/packages/browser-extension/src/sidepanel.ts @@ -23,12 +23,6 @@ export class Header extends LitElement { return this; } - async connectedCallback() { - super.connectedCallback(); - const resp = await fetch("https://genai.mariozechner.at/api/health"); - console.log(await resp.json()); - } - render() { return html`
@@ -48,9 +42,9 @@ export class Header extends LitElement { } const app = html` -
- - +
+ +
`; diff --git a/packages/browser-extension/src/state/agent-session.ts b/packages/browser-extension/src/state/agent-session.ts new file mode 100644 index 00000000..7e30174d --- /dev/null +++ b/packages/browser-extension/src/state/agent-session.ts @@ -0,0 +1,306 @@ +import type { Context } from "@mariozechner/pi-ai"; +import { + type AgentTool, + type AssistantMessage as AssistantMessageType, + getModel, + type ImageContent, + type Message, + type Model, + type TextContent, +} from "@mariozechner/pi-ai"; +import type { AppMessage } from "../Messages.js"; +import type { Attachment } from "../utils/attachment-utils.js"; +import { getAuthToken } from "../utils/auth-token.js"; +import { DirectTransport } from "./transports/DirectTransport.js"; +import { ProxyTransport } from "./transports/ProxyTransport.js"; +import type { AgentRunConfig, AgentTransport } from "./transports/types.js"; +import type { DebugLogEntry } from "./types.js"; + +export type ThinkingLevel = "off" | "minimal" | "low" | "medium" | "high"; + +export interface AgentSessionState { + id: string; + systemPrompt: string; + model: Model | null; + thinkingLevel: ThinkingLevel; + tools: AgentTool[]; + messages: AppMessage[]; + isStreaming: boolean; + streamMessage: Message | null; + pendingToolCalls: Set; + error?: string; +} + +export type AgentSessionEvent = + | { type: "state-update"; state: AgentSessionState } + | { type: "error-no-model" } + | { type: "error-no-api-key"; provider: string }; + +export type TransportMode = "direct" | "proxy"; + +export interface AgentSessionOptions { + initialState?: Partial; + messagePreprocessor?: (messages: AppMessage[]) => Promise; + debugListener?: (entry: DebugLogEntry) => void; + transportMode?: TransportMode; + authTokenProvider?: () => Promise; +} + +export class AgentSession { + private _state: AgentSessionState = { + id: "default", + systemPrompt: "", + model: getModel("google", "gemini-2.5-flash-lite-preview-06-17"), + thinkingLevel: "off", + tools: [], + messages: [], + isStreaming: false, + streamMessage: null, + pendingToolCalls: new Set(), + error: undefined, + }; + private listeners = new Set<(e: AgentSessionEvent) => void>(); + private abortController?: AbortController; + private transport: AgentTransport; + private messagePreprocessor?: (messages: AppMessage[]) => Promise; + private debugListener?: (entry: DebugLogEntry) => void; + + constructor(opts: AgentSessionOptions = {}) { + this._state = { ...this._state, ...opts.initialState }; + this.messagePreprocessor = opts.messagePreprocessor; + this.debugListener = opts.debugListener; + + const mode = opts.transportMode || "direct"; + + if (mode === "proxy") { + this.transport = new ProxyTransport(async () => this.preprocessMessages()); + } else { + this.transport = new DirectTransport(async () => this.preprocessMessages()); + } + } + + private async preprocessMessages(): Promise { + const filtered = this._state.messages.map((m) => { + if (m.role === "user") { + const { attachments, ...rest } = m as AppMessage & { attachments?: Attachment[] }; + return rest; + } + return m; + }); + return this.messagePreprocessor ? this.messagePreprocessor(filtered as AppMessage[]) : (filtered as Message[]); + } + + get state(): AgentSessionState { + return this._state; + } + + subscribe(fn: (e: AgentSessionEvent) => void): () => void { + this.listeners.add(fn); + fn({ type: "state-update", state: this._state }); + return () => this.listeners.delete(fn); + } + + // Mutators + setSystemPrompt(v: string) { + this.patch({ systemPrompt: v }); + } + setModel(m: Model | null) { + this.patch({ model: m }); + } + setThinkingLevel(l: ThinkingLevel) { + this.patch({ thinkingLevel: l }); + } + setTools(t: AgentTool[]) { + this.patch({ tools: t }); + } + replaceMessages(ms: AppMessage[]) { + this.patch({ messages: ms.slice() }); + } + appendMessage(m: AppMessage) { + this.patch({ messages: [...this._state.messages, m] }); + } + clearMessages() { + this.patch({ messages: [] }); + } + + abort() { + this.abortController?.abort(); + } + + async prompt(input: string, attachments?: Attachment[]) { + const model = this._state.model; + if (!model) { + this.emit({ type: "error-no-model" }); + return; + } + + // Build user message with attachments + const content: Array = [{ type: "text", text: input }]; + if (attachments?.length) { + for (const a of attachments) { + if (a.type === "image") { + content.push({ type: "image", data: a.content, mimeType: a.mimeType }); + } else if (a.type === "document" && a.extractedText) { + content.push({ + type: "text", + text: `\n\n[Document: ${a.fileName}]\n${a.extractedText}`, + isDocument: true, + } as TextContent); + } + } + } + + const userMessage: AppMessage = { + role: "user", + content, + attachments: attachments?.length ? attachments : undefined, + }; + + this.abortController = new AbortController(); + this.patch({ isStreaming: true, streamMessage: null, error: undefined }); + + const reasoning = + this._state.thinkingLevel === "off" + ? undefined + : this._state.thinkingLevel === "minimal" + ? "low" + : this._state.thinkingLevel; + const cfg: AgentRunConfig = { + systemPrompt: this._state.systemPrompt, + tools: this._state.tools, + model, + reasoning, + }; + + try { + let partial: Message | null = null; + let turnDebug: DebugLogEntry | null = null; + let turnStart = 0; + for await (const ev of this.transport.run(userMessage as Message, cfg, this.abortController.signal)) { + switch (ev.type) { + case "turn_start": { + turnStart = performance.now(); + // Build request context snapshot + const existing = this._state.messages as Message[]; + const ctx: Context = { + systemPrompt: this._state.systemPrompt, + messages: [...existing], + tools: this._state.tools, + }; + turnDebug = { + timestamp: new Date().toISOString(), + request: { + provider: cfg.model.provider, + model: cfg.model.id, + context: { ...ctx }, + }, + sseEvents: [], + }; + break; + } + case "message_start": + case "message_update": { + partial = ev.message; + // Collect SSE-like events for debug (drop heavy partial) + if (ev.type === "message_update" && ev.assistantMessageEvent && turnDebug) { + const copy: any = { ...ev.assistantMessageEvent }; + if (copy && "partial" in copy) delete copy.partial; + turnDebug.sseEvents.push(JSON.stringify(copy)); + if (!turnDebug.ttft) turnDebug.ttft = performance.now() - turnStart; + } + this.patch({ streamMessage: ev.message }); + break; + } + case "message_end": { + partial = null; + this.appendMessage(ev.message as AppMessage); + this.patch({ streamMessage: null }); + if (turnDebug) { + if (ev.message.role !== "assistant" && ev.message.role !== "toolResult") { + turnDebug.request.context.messages.push(ev.message); + } + if (ev.message.role === "assistant") turnDebug.response = ev.message as any; + } + break; + } + case "tool_execution_start": { + const s = new Set(this._state.pendingToolCalls); + s.add(ev.toolCallId); + this.patch({ pendingToolCalls: s }); + break; + } + case "tool_execution_end": { + const s = new Set(this._state.pendingToolCalls); + s.delete(ev.toolCallId); + this.patch({ pendingToolCalls: s }); + break; + } + case "turn_end": { + // finalize current turn + if (turnDebug) { + turnDebug.totalTime = performance.now() - turnStart; + this.debugListener?.(turnDebug); + turnDebug = null; + } + break; + } + case "agent_end": { + this.patch({ streamMessage: null }); + break; + } + } + } + + if (partial && partial.role === "assistant" && partial.content.length > 0) { + const onlyEmpty = !partial.content.some( + (c) => + (c.type === "thinking" && c.thinking.trim().length > 0) || + (c.type === "text" && c.text.trim().length > 0) || + (c.type === "toolCall" && c.name.trim().length > 0), + ); + if (!onlyEmpty) { + this.appendMessage(partial as AppMessage); + } else { + if (this.abortController?.signal.aborted) { + throw new Error("Request was aborted"); + } + } + } + } catch (err: any) { + if (String(err?.message || err) === "no-api-key") { + this.emit({ type: "error-no-api-key", provider: model.provider }); + } else { + const msg: AssistantMessageType = { + role: "assistant", + content: [{ type: "text", text: "" }], + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: this.abortController?.signal.aborted ? "aborted" : "error", + errorMessage: err?.message || String(err), + }; + this.appendMessage(msg as AppMessage); + this.patch({ error: err?.message || String(err) }); + } + } finally { + this.patch({ isStreaming: false, streamMessage: null, pendingToolCalls: new Set() }); + this.abortController = undefined; + } + } + + private patch(p: Partial): void { + this._state = { ...this._state, ...p }; + this.emit({ type: "state-update", state: this._state }); + } + + private emit(e: AgentSessionEvent) { + this.listeners.forEach((l) => l(e)); + } +} diff --git a/packages/browser-extension/src/state/transports/DirectTransport.ts b/packages/browser-extension/src/state/transports/DirectTransport.ts new file mode 100644 index 00000000..ecd6e734 --- /dev/null +++ b/packages/browser-extension/src/state/transports/DirectTransport.ts @@ -0,0 +1,32 @@ +import { type AgentContext, agentLoop, type Message, type PromptConfig, type UserMessage } from "@mariozechner/pi-ai"; +import { keyStore } from "../KeyStore.js"; +import type { AgentRunConfig, AgentTransport } from "./types.js"; + +export class DirectTransport implements AgentTransport { + constructor(private readonly getMessages: () => Promise) {} + + async *run(userMessage: Message, cfg: AgentRunConfig, signal?: AbortSignal) { + // Get API key from KeyStore + const apiKey = await keyStore.getKey(cfg.model.provider); + if (!apiKey) { + throw new Error("no-api-key"); + } + + const context: AgentContext = { + systemPrompt: cfg.systemPrompt, + messages: await this.getMessages(), + tools: cfg.tools, + }; + + const pc: PromptConfig = { + model: cfg.model, + reasoning: cfg.reasoning, + apiKey, + }; + + // Yield events from agentLoop + for await (const ev of agentLoop(userMessage as unknown as UserMessage, context, pc, signal)) { + yield ev; + } + } +} diff --git a/packages/browser-extension/src/state/transports/ProxyTransport.ts b/packages/browser-extension/src/state/transports/ProxyTransport.ts new file mode 100644 index 00000000..4049778f --- /dev/null +++ b/packages/browser-extension/src/state/transports/ProxyTransport.ts @@ -0,0 +1,358 @@ +import type { + AgentContext, + AssistantMessage, + AssistantMessageEvent, + Context, + Message, + Model, + PromptConfig, + SimpleStreamOptions, + ToolCall, + UserMessage, +} from "@mariozechner/pi-ai"; +import { agentLoop } from "@mariozechner/pi-ai"; +import { AssistantMessageEventStream } from "@mariozechner/pi-ai/dist/utils/event-stream.js"; +import { parseStreamingJson } from "@mariozechner/pi-ai/dist/utils/json-parse.js"; +import { clearAuthToken, getAuthToken } from "../../utils/auth-token.js"; +import { i18n } from "../../utils/i18n.js"; +import type { ProxyAssistantMessageEvent } from "./proxy-types.js"; +import type { AgentRunConfig, AgentTransport } from "./types.js"; + +/** + * Stream function that proxies through a server instead of calling providers directly. + * The server strips the partial field from delta events to reduce bandwidth. + * We reconstruct the partial message client-side. + */ +function streamSimpleProxy( + model: Model, + context: Context, + options: SimpleStreamOptions & { authToken: string }, + proxyUrl: string, +): AssistantMessageEventStream { + const stream = new AssistantMessageEventStream(); + + (async () => { + // Initialize the partial message that we'll build up from events + const partial: AssistantMessage = { + role: "assistant", + stopReason: "stop", + content: [], + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + }; + + let reader: ReadableStreamDefaultReader | undefined; + + // Set up abort handler to cancel the reader + const abortHandler = () => { + if (reader) { + reader.cancel("Request aborted by user").catch(() => {}); + } + }; + + if (options.signal) { + options.signal.addEventListener("abort", abortHandler); + } + + try { + const response = await fetch(`${proxyUrl}/api/stream`, { + method: "POST", + headers: { + Authorization: `Bearer ${options.authToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model, + context, + options: { + temperature: options.temperature, + maxTokens: options.maxTokens, + reasoning: options.reasoning, + // Don't send apiKey or signal - those are added server-side + }, + }), + signal: options.signal, + }); + + if (!response.ok) { + let errorMessage = `Proxy error: ${response.status} ${response.statusText}`; + try { + const errorData = await response.json(); + if (errorData.error) { + errorMessage = `Proxy error: ${errorData.error}`; + } + } catch { + // Couldn't parse error response, use default message + } + throw new Error(errorMessage); + } + + // Parse SSE stream + reader = response.body!.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + // Check if aborted after reading + if (options.signal?.aborted) { + throw new Error("Request aborted by user"); + } + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + + for (const line of lines) { + if (line.startsWith("data: ")) { + const data = line.slice(6).trim(); + if (data) { + const proxyEvent = JSON.parse(data) as ProxyAssistantMessageEvent; + let event: AssistantMessageEvent | undefined; + + // Handle different event types + // Server sends events with partial for non-delta events, + // and without partial for delta events + switch (proxyEvent.type) { + case "start": + event = { type: "start", partial }; + break; + + case "text_start": + partial.content[proxyEvent.contentIndex] = { + type: "text", + text: "", + }; + event = { type: "text_start", contentIndex: proxyEvent.contentIndex, partial }; + break; + + case "text_delta": { + const content = partial.content[proxyEvent.contentIndex]; + if (content?.type === "text") { + content.text += proxyEvent.delta; + event = { + type: "text_delta", + contentIndex: proxyEvent.contentIndex, + delta: proxyEvent.delta, + partial, + }; + } else { + throw new Error("Received text_delta for non-text content"); + } + break; + } + case "text_end": { + const content = partial.content[proxyEvent.contentIndex]; + if (content?.type === "text") { + content.textSignature = proxyEvent.contentSignature; + event = { + type: "text_end", + contentIndex: proxyEvent.contentIndex, + content: content.text, + partial, + }; + } else { + throw new Error("Received text_end for non-text content"); + } + break; + } + + case "thinking_start": + partial.content[proxyEvent.contentIndex] = { + type: "thinking", + thinking: "", + }; + event = { type: "thinking_start", contentIndex: proxyEvent.contentIndex, partial }; + break; + + case "thinking_delta": { + const content = partial.content[proxyEvent.contentIndex]; + if (content?.type === "thinking") { + content.thinking += proxyEvent.delta; + event = { + type: "thinking_delta", + contentIndex: proxyEvent.contentIndex, + delta: proxyEvent.delta, + partial, + }; + } else { + throw new Error("Received thinking_delta for non-thinking content"); + } + break; + } + + case "thinking_end": { + const content = partial.content[proxyEvent.contentIndex]; + if (content?.type === "thinking") { + content.thinkingSignature = proxyEvent.contentSignature; + event = { + type: "thinking_end", + contentIndex: proxyEvent.contentIndex, + content: content.thinking, + partial, + }; + } else { + throw new Error("Received thinking_end for non-thinking content"); + } + break; + } + + case "toolcall_start": + partial.content[proxyEvent.contentIndex] = { + type: "toolCall", + id: proxyEvent.id, + name: proxyEvent.toolName, + arguments: {}, + partialJson: "", + } satisfies ToolCall & { partialJson: string } as ToolCall; + event = { type: "toolcall_start", contentIndex: proxyEvent.contentIndex, partial }; + break; + + case "toolcall_delta": { + const content = partial.content[proxyEvent.contentIndex]; + if (content?.type === "toolCall") { + (content as any).partialJson += proxyEvent.delta; + content.arguments = parseStreamingJson((content as any).partialJson) || {}; + event = { + type: "toolcall_delta", + contentIndex: proxyEvent.contentIndex, + delta: proxyEvent.delta, + partial, + }; + partial.content[proxyEvent.contentIndex] = { ...content }; // Trigger reactivity + } else { + throw new Error("Received toolcall_delta for non-toolCall content"); + } + break; + } + + case "toolcall_end": { + const content = partial.content[proxyEvent.contentIndex]; + if (content?.type === "toolCall") { + delete (content as any).partialJson; + event = { + type: "toolcall_end", + contentIndex: proxyEvent.contentIndex, + toolCall: content, + partial, + }; + } + break; + } + + case "done": + partial.stopReason = proxyEvent.reason; + partial.usage = proxyEvent.usage; + event = { type: "done", reason: proxyEvent.reason, message: partial }; + break; + + case "error": + partial.stopReason = proxyEvent.reason; + partial.errorMessage = proxyEvent.errorMessage; + partial.usage = proxyEvent.usage; + event = { type: "error", reason: proxyEvent.reason, error: partial }; + break; + + default: { + // Exhaustive check + const _exhaustiveCheck: never = proxyEvent; + console.warn(`Unhandled event type: ${(proxyEvent as any).type}`); + break; + } + } + + // Push the event to stream + if (event) { + stream.push(event); + } else { + throw new Error("Failed to create event from proxy event"); + } + } + } + } + } + + // Check if aborted after reading + if (options.signal?.aborted) { + throw new Error("Request aborted by user"); + } + + stream.end(); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + if (errorMessage.toLowerCase().includes("proxy") && errorMessage.includes("Unauthorized")) { + clearAuthToken(); + } + partial.stopReason = options.signal?.aborted ? "aborted" : "error"; + partial.errorMessage = errorMessage; + stream.push({ + type: "error", + reason: partial.stopReason, + error: partial, + } satisfies AssistantMessageEvent); + stream.end(); + } finally { + // Clean up abort handler + if (options.signal) { + options.signal.removeEventListener("abort", abortHandler); + } + } + })(); + + return stream; +} + +// Proxy transport executes the turn using a remote proxy server +export class ProxyTransport implements AgentTransport { + // Hardcoded proxy URL for now - will be made configurable later + private readonly proxyUrl = "https://genai.mariozechner.at"; + + constructor(private readonly getMessages: () => Promise) {} + + async *run(userMessage: Message, cfg: AgentRunConfig, signal?: AbortSignal) { + const authToken = await getAuthToken(); + if (!authToken) { + throw new Error(i18n("Auth token is required for proxy transport")); + } + + // Use proxy - no local API key needed + const streamFn = (model: Model, context: Context, options: SimpleStreamOptions | undefined) => { + return streamSimpleProxy( + model, + context, + { + ...options, + authToken, + }, + this.proxyUrl, + ); + }; + + const context: AgentContext = { + systemPrompt: cfg.systemPrompt, + messages: await this.getMessages(), + tools: cfg.tools, + }; + + const pc: PromptConfig = { + model: cfg.model, + reasoning: cfg.reasoning, + }; + + // Yield events from the upstream agentLoop iterator + // Pass streamFn as the 5th parameter to use proxy + for await (const ev of agentLoop(userMessage as unknown as UserMessage, context, pc, signal, streamFn)) { + yield ev; + } + } +} diff --git a/packages/browser-extension/src/state/transports/index.ts b/packages/browser-extension/src/state/transports/index.ts new file mode 100644 index 00000000..1d91e36f --- /dev/null +++ b/packages/browser-extension/src/state/transports/index.ts @@ -0,0 +1,3 @@ +export * from "./DirectTransport.js"; +export * from "./ProxyTransport.js"; +export * from "./types.js"; diff --git a/packages/browser-extension/src/state/transports/proxy-types.ts b/packages/browser-extension/src/state/transports/proxy-types.ts new file mode 100644 index 00000000..94d4dbf9 --- /dev/null +++ b/packages/browser-extension/src/state/transports/proxy-types.ts @@ -0,0 +1,15 @@ +import type { StopReason, Usage } from "@mariozechner/pi-ai"; + +export type ProxyAssistantMessageEvent = + | { type: "start" } + | { type: "text_start"; contentIndex: number } + | { type: "text_delta"; contentIndex: number; delta: string } + | { type: "text_end"; contentIndex: number; contentSignature?: string } + | { type: "thinking_start"; contentIndex: number } + | { type: "thinking_delta"; contentIndex: number; delta: string } + | { type: "thinking_end"; contentIndex: number; contentSignature?: string } + | { type: "toolcall_start"; contentIndex: number; id: string; toolName: string } + | { type: "toolcall_delta"; contentIndex: number; delta: string } + | { type: "toolcall_end"; contentIndex: number } + | { type: "done"; reason: Extract; usage: Usage } + | { type: "error"; reason: Extract; errorMessage: string; usage: Usage }; diff --git a/packages/browser-extension/src/state/transports/types.ts b/packages/browser-extension/src/state/transports/types.ts new file mode 100644 index 00000000..8f432ce6 --- /dev/null +++ b/packages/browser-extension/src/state/transports/types.ts @@ -0,0 +1,16 @@ +import type { AgentEvent, AgentTool, Message, Model } from "@mariozechner/pi-ai"; + +// The minimal configuration needed to run a turn. +export interface AgentRunConfig { + systemPrompt: string; + tools: AgentTool[]; + model: Model; + reasoning?: "low" | "medium" | "high"; +} + +// Events yielded by transports must match the @mariozechner/pi-ai prompt() events. +// We re-export the Message type above; consumers should use the upstream AgentEvent type. + +export interface AgentTransport { + run(userMessage: Message, config: AgentRunConfig, signal?: AbortSignal): AsyncIterable; // passthrough of AgentEvent from upstream +} diff --git a/packages/browser-extension/src/state/types.ts b/packages/browser-extension/src/state/types.ts new file mode 100644 index 00000000..c5513941 --- /dev/null +++ b/packages/browser-extension/src/state/types.ts @@ -0,0 +1,11 @@ +import type { AssistantMessage, Context } from "@mariozechner/pi-ai"; + +export interface DebugLogEntry { + timestamp: string; + request: { provider: string; model: string; context: Context }; + response?: AssistantMessage; + error?: unknown; + sseEvents: string[]; + ttft?: number; + totalTime?: number; +} diff --git a/packages/browser-extension/src/tools/index.ts b/packages/browser-extension/src/tools/index.ts new file mode 100644 index 00000000..e24aa90d --- /dev/null +++ b/packages/browser-extension/src/tools/index.ts @@ -0,0 +1,38 @@ +import { html, type TemplateResult } from "@mariozechner/mini-lit"; +import type { ToolResultMessage } from "@mariozechner/pi-ai"; +import { getToolRenderer, registerToolRenderer } from "./renderer-registry.js"; +import { BashRenderer } from "./renderers/BashRenderer.js"; +import { CalculateRenderer } from "./renderers/CalculateRenderer.js"; +import { DefaultRenderer } from "./renderers/DefaultRenderer.js"; +import { GetCurrentTimeRenderer } from "./renderers/GetCurrentTimeRenderer.js"; + +// Register all built-in tool renderers +registerToolRenderer("calculate", new CalculateRenderer()); +registerToolRenderer("get_current_time", new GetCurrentTimeRenderer()); +registerToolRenderer("bash", new BashRenderer()); + +const defaultRenderer = new DefaultRenderer(); + +/** + * Render tool call parameters + */ +export function renderToolParams(toolName: string, params: any, isStreaming?: boolean): TemplateResult { + const renderer = getToolRenderer(toolName); + if (renderer) { + return renderer.renderParams(params, isStreaming); + } + return defaultRenderer.renderParams(params, isStreaming); +} + +/** + * Render tool result + */ +export function renderToolResult(toolName: string, params: any, result: ToolResultMessage): TemplateResult { + const renderer = getToolRenderer(toolName); + if (renderer) { + return renderer.renderResult(params, result); + } + return defaultRenderer.renderResult(params, result); +} + +export { registerToolRenderer, getToolRenderer }; diff --git a/packages/browser-extension/src/tools/renderer-registry.ts b/packages/browser-extension/src/tools/renderer-registry.ts new file mode 100644 index 00000000..c6e3b71b --- /dev/null +++ b/packages/browser-extension/src/tools/renderer-registry.ts @@ -0,0 +1,18 @@ +import type { ToolRenderer } from "./types.js"; + +// Registry of tool renderers +export const toolRenderers = new Map(); + +/** + * Register a custom tool renderer + */ +export function registerToolRenderer(toolName: string, renderer: ToolRenderer): void { + toolRenderers.set(toolName, renderer); +} + +/** + * Get a tool renderer by name + */ +export function getToolRenderer(toolName: string): ToolRenderer | undefined { + return toolRenderers.get(toolName); +} diff --git a/packages/browser-extension/src/tools/renderers/BashRenderer.ts b/packages/browser-extension/src/tools/renderers/BashRenderer.ts new file mode 100644 index 00000000..ba9ae330 --- /dev/null +++ b/packages/browser-extension/src/tools/renderers/BashRenderer.ts @@ -0,0 +1,45 @@ +import { html, type TemplateResult } from "@mariozechner/mini-lit"; +import type { ToolResultMessage } from "@mariozechner/pi-ai"; +import { i18n } from "../../utils/i18n.js"; +import type { ToolRenderer } from "../types.js"; + +interface BashParams { + command: string; +} + +// Bash tool has undefined details (only uses output) +export class BashRenderer implements ToolRenderer { + renderParams(params: BashParams, isStreaming?: boolean): TemplateResult { + if (isStreaming && (!params.command || params.command.length === 0)) { + return html`
${i18n("Writing command...")}
`; + } + + return html` +
+ ${i18n("Running command:")} + ${params.command} +
+ `; + } + + renderResult(_params: BashParams, result: ToolResultMessage): TemplateResult { + const output = result.output || ""; + const isError = result.isError === true; + + if (isError) { + return html` +
+
${i18n("Command failed:")}
+ 
${output}
+
+ `; + } + + // Display the command output + return html` +
+ 
${output}
+
+ `; + } +} diff --git a/packages/browser-extension/src/tools/renderers/CalculateRenderer.ts b/packages/browser-extension/src/tools/renderers/CalculateRenderer.ts new file mode 100644 index 00000000..fb4eec05 --- /dev/null +++ b/packages/browser-extension/src/tools/renderers/CalculateRenderer.ts @@ -0,0 +1,49 @@ +import { html, type TemplateResult } from "@mariozechner/mini-lit"; +import type { ToolResultMessage } from "@mariozechner/pi-ai"; +import { i18n } from "../../utils/i18n.js"; +import type { ToolRenderer } from "../types.js"; + +interface CalculateParams { + expression: string; +} + +// Calculate tool has undefined details (only uses output) +export class CalculateRenderer implements ToolRenderer { + renderParams(params: CalculateParams, isStreaming?: boolean): TemplateResult { + if (isStreaming && !params.expression) { + return html`
${i18n("Writing expression...")}
`; + } + + return html` +
+ ${i18n("Calculating")} + ${params.expression} +
+ `; + } + + renderResult(_params: CalculateParams, result: ToolResultMessage): TemplateResult { + // Parse the output to make it look nicer + const output = result.output || ""; + const isError = result.isError === true; + + if (isError) { + return html`
${output}
`; + } + + // Try to split on = to show expression and result separately + const parts = output.split(" = "); + if (parts.length === 2) { + return html` +
+ ${parts[0]} + = + ${parts[1]} +
+ `; + } + + // Fallback to showing the whole output + return html`
${output}
`; + } +} diff --git a/packages/browser-extension/src/tools/renderers/DefaultRenderer.ts b/packages/browser-extension/src/tools/renderers/DefaultRenderer.ts new file mode 100644 index 00000000..dd15e028 --- /dev/null +++ b/packages/browser-extension/src/tools/renderers/DefaultRenderer.ts @@ -0,0 +1,36 @@ +import { html, type TemplateResult } from "@mariozechner/mini-lit"; +import type { ToolResultMessage } from "@mariozechner/pi-ai"; +import { i18n } from "../../utils/i18n.js"; +import type { ToolRenderer } from "../types.js"; + +export class DefaultRenderer implements ToolRenderer { + renderParams(params: any, isStreaming?: boolean): TemplateResult { + let text: string; + let isJson = false; + + try { + text = JSON.stringify(JSON.parse(params), null, 2); + isJson = true; + } catch { + try { + text = JSON.stringify(params, null, 2); + isJson = true; + } catch { + text = String(params); + } + } + + if (isStreaming && (!text || text === "{}" || text === "null")) { + return html`
${i18n("Preparing tool parameters...")}
`; + } + + return html``; + } + + renderResult(_params: any, result: ToolResultMessage): TemplateResult { + // Just show the output field - that's what was sent to the LLM + const text = result.output || i18n("(no output)"); + + return html`
${text}
`; + } +} diff --git a/packages/browser-extension/src/tools/renderers/GetCurrentTimeRenderer.ts b/packages/browser-extension/src/tools/renderers/GetCurrentTimeRenderer.ts new file mode 100644 index 00000000..db2d629b --- /dev/null +++ b/packages/browser-extension/src/tools/renderers/GetCurrentTimeRenderer.ts @@ -0,0 +1,39 @@ +import { html, type TemplateResult } from "@mariozechner/mini-lit"; +import type { ToolResultMessage } from "@mariozechner/pi-ai"; +import { i18n } from "../../utils/i18n.js"; +import type { ToolRenderer } from "../types.js"; + +interface GetCurrentTimeParams { + timezone?: string; +} + +// GetCurrentTime tool has undefined details (only uses output) +export class GetCurrentTimeRenderer implements ToolRenderer { + renderParams(params: GetCurrentTimeParams, isStreaming?: boolean): TemplateResult { + if (params.timezone) { + return html` +
+ ${i18n("Getting current time in")} + ${params.timezone} +
+ `; + } + return html` +
+ ${i18n("Getting current date and time")}${isStreaming ? "..." : ""} +
+ `; + } + + renderResult(_params: GetCurrentTimeParams, result: ToolResultMessage): TemplateResult { + const output = result.output || ""; + const isError = result.isError === true; + + if (isError) { + return html`
${output}
`; + } + + // Display the date/time result + return html`
${output}
`; + } +} diff --git a/packages/browser-extension/src/tools/types.ts b/packages/browser-extension/src/tools/types.ts new file mode 100644 index 00000000..6db8e8ae --- /dev/null +++ b/packages/browser-extension/src/tools/types.ts @@ -0,0 +1,7 @@ +import type { ToolResultMessage } from "@mariozechner/pi-ai"; +import type { TemplateResult } from "lit"; + +export interface ToolRenderer { + renderParams(params: TParams, isStreaming?: boolean): TemplateResult; + renderResult(params: TParams, result: ToolResultMessage): TemplateResult; +} diff --git a/packages/browser-extension/src/utils/auth-token.ts b/packages/browser-extension/src/utils/auth-token.ts new file mode 100644 index 00000000..776fcfbc --- /dev/null +++ b/packages/browser-extension/src/utils/auth-token.ts @@ -0,0 +1,22 @@ +import { PromptDialog } from "../dialogs/PromptDialog.js"; +import { i18n } from "./i18n.js"; + +export async function getAuthToken(): Promise { + let authToken: string | undefined = localStorage.getItem(`auth-token`) || ""; + if (authToken) return authToken; + + while (true) { + authToken = ( + await PromptDialog.ask(i18n("Enter Auth Token"), i18n("Please enter your auth token."), "", true) + )?.trim(); + if (authToken) { + localStorage.setItem(`auth-token`, authToken); + break; + } + } + return authToken?.trim() || undefined; +} + +export async function clearAuthToken() { + localStorage.removeItem(`auth-token`); +} diff --git a/packages/browser-extension/src/utils/i18n.ts b/packages/browser-extension/src/utils/i18n.ts index 6edead88..ee252b8c 100644 --- a/packages/browser-extension/src/utils/i18n.ts +++ b/packages/browser-extension/src/utils/i18n.ts @@ -44,6 +44,30 @@ declare module "@mariozechner/mini-lit" { "No content available": string; "Failed to display text content": string; "API keys are required to use AI models. Get your keys from the provider's website.": string; + console: string; + "Copy output": string; + "Copied!": string; + "Error:": string; + "Request aborted": string; + Call: string; + Result: string; + "(no result)": string; + "Waiting for tool result…": string; + "Call was aborted; no result.": string; + "No session available": string; + "No session set": string; + "Preparing tool parameters...": string; + "(no output)": string; + "Writing expression...": string; + Calculating: string; + "Getting current time in": string; + "Getting current date and time": string; + "Writing command...": string; + "Running command:": string; + "Command failed:": string; + "Enter Auth Token": string; + "Please enter your auth token.": string; + "Auth token is required for proxy transport": string; } } @@ -94,6 +118,30 @@ const translations = { "Failed to display text content": "Failed to display text content", "API keys are required to use AI models. Get your keys from the provider's website.": "API keys are required to use AI models. Get your keys from the provider's website.", + console: "console", + "Copy output": "Copy output", + "Copied!": "Copied!", + "Error:": "Error:", + "Request aborted": "Request aborted", + Call: "Call", + Result: "Result", + "(no result)": "(no result)", + "Waiting for tool result…": "Waiting for tool result…", + "Call was aborted; no result.": "Call was aborted; no result.", + "No session available": "No session available", + "No session set": "No session set", + "Preparing tool parameters...": "Preparing tool parameters...", + "(no output)": "(no output)", + "Writing expression...": "Writing expression...", + Calculating: "Calculating", + "Getting current time in": "Getting current time in", + "Getting current date and time": "Getting current date and time", + "Writing command...": "Writing command...", + "Running command:": "Running command:", + "Command failed:": "Command failed:", + "Enter Auth Token": "Enter Auth Token", + "Please enter your auth token.": "Please enter your auth token.", + "Auth token is required for proxy transport": "Auth token is required for proxy transport", }, de: { ...defaultGerman, @@ -141,6 +189,30 @@ const translations = { "Failed to display text content": "Textinhalt konnte nicht angezeigt werden", "API keys are required to use AI models. Get your keys from the provider's website.": "API-Schlüssel sind erforderlich, um KI-Modelle zu verwenden. Holen Sie sich Ihre Schlüssel von der Website des Anbieters.", + console: "Konsole", + "Copy output": "Ausgabe kopieren", + "Copied!": "Kopiert!", + "Error:": "Fehler:", + "Request aborted": "Anfrage abgebrochen", + Call: "Aufruf", + Result: "Ergebnis", + "(no result)": "(kein Ergebnis)", + "Waiting for tool result…": "Warte auf Tool-Ergebnis…", + "Call was aborted; no result.": "Aufruf wurde abgebrochen; kein Ergebnis.", + "No session available": "Keine Sitzung verfügbar", + "No session set": "Keine Sitzung gesetzt", + "Preparing tool parameters...": "Bereite Tool-Parameter vor...", + "(no output)": "(keine Ausgabe)", + "Writing expression...": "Schreibe Ausdruck...", + Calculating: "Berechne", + "Getting current time in": "Hole aktuelle Zeit in", + "Getting current date and time": "Hole aktuelles Datum und Uhrzeit", + "Writing command...": "Schreibe Befehl...", + "Running command:": "Führe Befehl aus:", + "Command failed:": "Befehl fehlgeschlagen:", + "Enter Auth Token": "Auth-Token eingeben", + "Please enter your auth token.": "Bitte geben Sie Ihr Auth-Token ein.", + "Auth token is required for proxy transport": "Auth-Token ist für Proxy-Transport erforderlich", }, };