diff --git a/packages/web-ui/example/src/custom-messages.ts b/packages/web-ui/example/src/custom-messages.ts index 8cb3bdfe..31c5d3fd 100644 --- a/packages/web-ui/example/src/custom-messages.ts +++ b/packages/web-ui/example/src/custom-messages.ts @@ -1,6 +1,6 @@ import { Alert } from "@mariozechner/mini-lit/dist/Alert.js"; -import type { Message } from "@mariozechner/pi-ai"; -import type { AppMessage, MessageRenderer } from "@mariozechner/pi-web-ui"; +import type { ImageContent, Message, TextContent } from "@mariozechner/pi-ai"; +import type { AgentMessage, Attachment, MessageRenderer, UserMessageWithAttachments } from "@mariozechner/pi-web-ui"; import { registerMessageRenderer } from "@mariozechner/pi-web-ui"; import { html } from "lit"; @@ -16,8 +16,9 @@ export interface SystemNotificationMessage { timestamp: string; } -// Extend CustomMessages interface via declaration merging -declare module "@mariozechner/pi-web-ui" { +// Extend CustomAgentMessages interface via declaration merging +// This must target pi-agent-core where CustomAgentMessages is defined +declare module "@mariozechner/pi-agent-core" { interface CustomAgentMessages { "system-notification": SystemNotificationMessage; } @@ -74,8 +75,28 @@ export function createSystemNotification( // 5. CUSTOM MESSAGE TRANSFORMER // ============================================================================ -// Transform custom messages to user messages with tags so LLM can see them -export function customMessageTransformer(messages: AppMessage[]): Message[] { +// Convert attachments to content blocks +function convertAttachments(attachments: Attachment[]): (TextContent | ImageContent)[] { + const content: (TextContent | ImageContent)[] = []; + for (const attachment of attachments) { + if (attachment.type === "image") { + content.push({ + type: "image", + data: attachment.content, + mimeType: attachment.mimeType, + } as ImageContent); + } else if (attachment.type === "document" && attachment.extractedText) { + content.push({ + type: "text", + text: `\n\n[Document: ${attachment.fileName}]\n${attachment.extractedText}`, + } as TextContent); + } + } + return content; +} + +// Transform custom messages to LLM-compatible messages +export function customMessageTransformer(messages: AgentMessage[]): Message[] { return messages .filter((m) => { // Filter out artifact messages - they're for session reconstruction only @@ -85,7 +106,11 @@ export function customMessageTransformer(messages: AppMessage[]): Message[] { // Keep LLM-compatible messages + custom messages return ( - m.role === "user" || m.role === "assistant" || m.role === "toolResult" || m.role === "system-notification" + m.role === "user" || + m.role === "user-with-attachments" || + m.role === "assistant" || + m.role === "toolResult" || + m.role === "system-notification" ); }) .map((m) => { @@ -95,13 +120,25 @@ export function customMessageTransformer(messages: AppMessage[]): Message[] { return { role: "user", content: `${notification.message}`, + timestamp: Date.now(), } as Message; } - // Strip attachments from user messages - if (m.role === "user-with-attachment") { - const { attachments: _, ...rest } = m; - return rest as Message; + // Convert user-with-attachments to user message with content blocks + if (m.role === "user-with-attachments") { + const msg = m as UserMessageWithAttachments; + const textContent: (TextContent | ImageContent)[] = + typeof msg.content === "string" ? [{ type: "text", text: msg.content }] : [...msg.content]; + + if (msg.attachments) { + textContent.push(...convertAttachments(msg.attachments)); + } + + return { + role: "user", + content: textContent, + timestamp: msg.timestamp, + } as Message; } return m as Message; diff --git a/packages/web-ui/example/src/main.ts b/packages/web-ui/example/src/main.ts index ef49faba..376d6b36 100644 --- a/packages/web-ui/example/src/main.ts +++ b/packages/web-ui/example/src/main.ts @@ -1,10 +1,9 @@ import "@mariozechner/mini-lit/dist/ThemeToggle.js"; +import { Agent, type AgentMessage } from "@mariozechner/pi-agent-core"; import { getModel } from "@mariozechner/pi-ai"; import { - Agent, type AgentState, ApiKeyPromptDialog, - type AppMessage, AppStorage, ChatPanel, CustomProvidersStore, @@ -13,7 +12,6 @@ import { // PersistentStorageDialog, // TODO: Fix - currently broken ProviderKeysStore, ProvidersModelsTab, - ProviderTransport, ProxyTab, SessionListDialog, SessionsStore, @@ -75,7 +73,7 @@ let agent: Agent; let chatPanel: ChatPanel; let agentUnsubscribe: (() => void) | undefined; -const generateTitle = (messages: AppMessage[]): string => { +const generateTitle = (messages: AgentMessage[]): string => { const firstUserMsg = messages.find((m) => m.role === "user"); if (!firstUserMsg || firstUserMsg.role !== "user") return ""; @@ -99,7 +97,7 @@ const generateTitle = (messages: AppMessage[]): string => { return text.length <= 50 ? text : `${text.substring(0, 47)}...`; }; -const shouldSaveSession = (messages: AppMessage[]): boolean => { +const shouldSaveSession = (messages: AgentMessage[]): boolean => { const hasUserMsg = messages.some((m: any) => m.role === "user"); const hasAssistantMsg = messages.some((m: any) => m.role === "assistant"); return hasUserMsg && hasAssistantMsg; @@ -166,8 +164,6 @@ const createAgent = async (initialState?: Partial) => { agentUnsubscribe(); } - const transport = new ProviderTransport(); - agent = new Agent({ initialState: initialState || { systemPrompt: `You are a helpful AI assistant with access to various tools. @@ -182,9 +178,8 @@ Feel free to use these tools when needed to provide accurate and helpful respons messages: [], tools: [], }, - transport, - // Custom transformer: convert system notifications to user messages with tags - messageTransformer: customMessageTransformer, + // Custom transformer: convert custom messages to LLM-compatible format + convertToLlm: customMessageTransformer, }); agentUnsubscribe = agent.subscribe((event: any) => { diff --git a/packages/web-ui/example/tsconfig.json b/packages/web-ui/example/tsconfig.json index 340e2439..e095a279 100644 --- a/packages/web-ui/example/tsconfig.json +++ b/packages/web-ui/example/tsconfig.json @@ -6,6 +6,7 @@ "moduleResolution": "bundler", "paths": { "*": ["./*"], + "@mariozechner/pi-agent-core": ["../../agent/dist/index.d.ts"], "@mariozechner/pi-ai": ["../../ai/dist/index.d.ts"], "@mariozechner/pi-tui": ["../../tui/dist/index.d.ts"], "@mariozechner/pi-web-ui": ["../dist/index.d.ts"] diff --git a/packages/web-ui/src/components/AgentInterface.ts b/packages/web-ui/src/components/AgentInterface.ts index 6ee33126..210ea714 100644 --- a/packages/web-ui/src/components/AgentInterface.ts +++ b/packages/web-ui/src/components/AgentInterface.ts @@ -12,6 +12,7 @@ import type { Agent, AgentEvent } from "@mariozechner/pi-agent-core"; import type { Attachment } from "../utils/attachment-utils.js"; import { formatUsage } from "../utils/format.js"; import { i18n } from "../utils/i18n.js"; +import type { UserMessageWithAttachments } from "./Messages.js"; import type { StreamingMessageContainer } from "./StreamingMessageContainer.js"; @customElement("agent-interface") @@ -202,7 +203,18 @@ export class AgentInterface extends LitElement { this._messageEditor.attachments = []; this._autoScroll = true; // Enable auto-scroll when sending a message - await this.session?.prompt(input, attachments); + // Compose message with attachments if any + if (attachments && attachments.length > 0) { + const message: UserMessageWithAttachments = { + role: "user-with-attachments", + content: input, + attachments, + timestamp: Date.now(), + }; + await this.session?.prompt(message); + } else { + await this.session?.prompt(input); + } } private renderMessages() { diff --git a/packages/web-ui/src/components/MessageEditor.ts b/packages/web-ui/src/components/MessageEditor.ts index ae286d05..78e44230 100644 --- a/packages/web-ui/src/components/MessageEditor.ts +++ b/packages/web-ui/src/components/MessageEditor.ts @@ -9,6 +9,7 @@ import { Brain, Loader2, Paperclip, Send, Sparkles, Square } from "lucide"; import { type Attachment, loadAttachment } from "../utils/attachment-utils.js"; import { i18n } from "../utils/i18n.js"; import "./AttachmentTile.js"; +import type { ThinkingLevel } from "@mariozechner/pi-agent-core"; @customElement("message-editor") export class MessageEditor extends LitElement { @@ -28,7 +29,7 @@ export class MessageEditor extends LitElement { @property() isStreaming = false; @property() currentModel?: Model; - @property() thinkingLevel: "off" | "minimal" | "low" | "medium" | "high" = "off"; + @property() thinkingLevel: ThinkingLevel = "off"; @property() showAttachmentButton = true; @property() showModelSelector = true; @property() showThinkingSelector = true; diff --git a/packages/web-ui/src/components/Messages.ts b/packages/web-ui/src/components/Messages.ts index 5588005c..35637a30 100644 --- a/packages/web-ui/src/components/Messages.ts +++ b/packages/web-ui/src/components/Messages.ts @@ -34,7 +34,7 @@ export interface ArtifactMessage { declare module "@mariozechner/pi-agent-core" { interface CustomAgentMessages { - "user-with-attachment": UserMessageWithAttachments; + "user-with-attachments": UserMessageWithAttachments; artifact: ArtifactMessage; } }