diff --git a/packages/ai/src/providers/anthropic.ts b/packages/ai/src/providers/anthropic.ts index 2a1d9348..ccae567b 100644 --- a/packages/ai/src/providers/anthropic.ts +++ b/packages/ai/src/providers/anthropic.ts @@ -77,10 +77,15 @@ export class AnthropicLLM implements LLM { return this.modelInfo; } + getApi(): string { + return "anthropic-messages"; + } + async generate(context: Context, options?: AnthropicLLMOptions): Promise { const output: AssistantMessage = { role: "assistant", content: [], + api: this.getApi(), provider: this.modelInfo.provider, model: this.modelInfo.id, usage: { @@ -260,7 +265,7 @@ export class AnthropicLLM implements LLM { const params: MessageParam[] = []; // Transform messages for cross-provider compatibility - const transformedMessages = transformMessages(messages, this.modelInfo); + const transformedMessages = transformMessages(messages, this.modelInfo, this.getApi()); for (const msg of transformedMessages) { if (msg.role === "user") { @@ -290,9 +295,12 @@ export class AnthropicLLM implements LLM { }; } }); + const filteredBlocks = !this.modelInfo?.input.includes("image") + ? blocks.filter((b) => b.type !== "image") + : blocks; params.push({ role: "user", - content: blocks, + content: filteredBlocks, }); } } else if (msg.role === "assistant") { diff --git a/packages/ai/src/providers/google.ts b/packages/ai/src/providers/google.ts index d5a5761c..c304cf84 100644 --- a/packages/ai/src/providers/google.ts +++ b/packages/ai/src/providers/google.ts @@ -6,6 +6,7 @@ import { type GenerateContentParameters, GoogleGenAI, type Part, + setDefaultBaseUrls, } from "@google/genai"; import { calculateCost } from "../models.js"; import type { @@ -52,10 +53,15 @@ export class GoogleLLM implements LLM { return this.modelInfo; } + getApi(): string { + return "google-generative-ai"; + } + async generate(context: Context, options?: GoogleLLMOptions): Promise { const output: AssistantMessage = { role: "assistant", content: [], + api: this.getApi(), provider: this.modelInfo.provider, model: this.modelInfo.id, usage: { @@ -247,7 +253,7 @@ export class GoogleLLM implements LLM { const contents: Content[] = []; // Transform messages for cross-provider compatibility - const transformedMessages = transformMessages(messages, this.modelInfo); + const transformedMessages = transformMessages(messages, this.modelInfo, this.getApi()); for (const msg of transformedMessages) { if (msg.role === "user") { @@ -272,9 +278,12 @@ export class GoogleLLM implements LLM { }; } }); + const filteredParts = !this.modelInfo?.input.includes("image") + ? parts.filter((p) => p.text !== undefined) + : parts; contents.push({ role: "user", - parts, + parts: filteredParts, }); } } else if (msg.role === "assistant") { diff --git a/packages/ai/src/providers/openai-completions.ts b/packages/ai/src/providers/openai-completions.ts index 0696875a..a1ad6fad 100644 --- a/packages/ai/src/providers/openai-completions.ts +++ b/packages/ai/src/providers/openai-completions.ts @@ -48,10 +48,15 @@ export class OpenAICompletionsLLM implements LLM { return this.modelInfo; } + getApi(): string { + return "openai-completions"; + } + async generate(request: Context, options?: OpenAICompletionsLLMOptions): Promise { const output: AssistantMessage = { role: "assistant", content: [], + api: this.getApi(), provider: this.modelInfo.provider, model: this.modelInfo.id, usage: { @@ -313,7 +318,7 @@ export class OpenAICompletionsLLM implements LLM { const params: ChatCompletionMessageParam[] = []; // Transform messages for cross-provider compatibility - const transformedMessages = transformMessages(messages, this.modelInfo); + const transformedMessages = transformMessages(messages, this.modelInfo, this.getApi()); // Add system prompt if provided if (systemPrompt) { @@ -353,9 +358,12 @@ export class OpenAICompletionsLLM implements LLM { } satisfies ChatCompletionContentPartImage; } }); + const filteredContent = !this.modelInfo?.input.includes("image") + ? content.filter((c) => c.type !== "image_url") + : content; params.push({ role: "user", - content, + content: filteredContent, }); } } else if (msg.role === "assistant") { diff --git a/packages/ai/src/providers/openai-responses.ts b/packages/ai/src/providers/openai-responses.ts index b3afe6ef..271082ab 100644 --- a/packages/ai/src/providers/openai-responses.ts +++ b/packages/ai/src/providers/openai-responses.ts @@ -51,10 +51,15 @@ export class OpenAIResponsesLLM implements LLM { return this.modelInfo; } + getApi(): string { + return "openai-responses"; + } + async generate(request: Context, options?: OpenAIResponsesLLMOptions): Promise { const output: AssistantMessage = { role: "assistant", content: [], + api: this.getApi(), provider: this.modelInfo.provider, model: this.modelInfo.id, usage: { @@ -287,7 +292,7 @@ export class OpenAIResponsesLLM implements LLM { const input: ResponseInput = []; // Transform messages for cross-provider compatibility - const transformedMessages = transformMessages(messages, this.modelInfo); + const transformedMessages = transformMessages(messages, this.modelInfo, this.getApi()); // Add system prompt if provided if (systemPrompt) { @@ -324,9 +329,12 @@ export class OpenAIResponsesLLM implements LLM { } satisfies ResponseInputImage; } }); + const filteredContent = !this.modelInfo?.input.includes("image") + ? content.filter((c) => c.type !== "input_image") + : content; input.push({ role: "user", - content, + content: filteredContent, }); } } else if (msg.role === "assistant") { diff --git a/packages/ai/src/providers/utils.ts b/packages/ai/src/providers/utils.ts index db751fff..b42c7c0d 100644 --- a/packages/ai/src/providers/utils.ts +++ b/packages/ai/src/providers/utils.ts @@ -12,7 +12,7 @@ import type { AssistantMessage, Message, Model } from "../types.js"; * @param model The target model that will process these messages * @returns A copy of the messages array with transformations applied */ -export function transformMessages(messages: Message[], model: Model): Message[] { +export function transformMessages(messages: Message[], model: Model, api: string): Message[] { return messages.map((msg) => { // User and toolResult messages pass through unchanged if (msg.role === "user" || msg.role === "toolResult") { @@ -23,8 +23,8 @@ export function transformMessages(messages: Message[], model: Model): Message[] if (msg.role === "assistant") { const assistantMsg = msg as AssistantMessage; - // If message is from the same provider and model, keep as-is - if (assistantMsg.provider === model.provider && assistantMsg.model === model.id) { + // If message is from the same provider and API, keep as is + if (assistantMsg.provider === model.provider && assistantMsg.api === api) { return msg; } diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 5ad51b00..8a17ddd2 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -8,6 +8,7 @@ export interface LLMOptions { export interface LLM { generate(request: Context, options?: T): Promise; getModel(): Model; + getApi(): string; } export interface TextContent { @@ -59,6 +60,7 @@ export interface UserMessage { export interface AssistantMessage { role: "assistant"; content: (TextContent | ThinkingContent | ToolCall)[]; + api: string; provider: string; model: string; usage: Usage;