fix(ai): correct Gemini tool result format and improve type safety

- Fix tool result format for Gemini 3 Flash Preview compatibility
  - Use 'output' key for successful results (not 'result')
  - Use 'error' key for error results (not 'isError')
  - Per Google SDK documentation for FunctionResponse.response

- Improve type safety in google.ts provider
  - Add ImageContent import and use proper type guards
  - Replace 'as any' casts with proper typing
  - Import and use Schema type for tool parameters
  - Add proper typing for index deletion in error handler

- Add comprehensive test for Gemini 3 Flash tool calling
  - Tests successful tool call and result handling
  - Tests error tool result handling
  - Verifies fix for issue #213

Fixes #213
This commit is contained in:
Mario Zechner 2025-12-18 13:43:09 +00:00
parent de2de851c8
commit 84018b0707
2 changed files with 191 additions and 16 deletions

View file

@ -6,6 +6,7 @@ import {
type GenerateContentParameters,
GoogleGenAI,
type Part,
type Schema,
type ThinkingConfig,
type ThinkingLevel,
} from "@google/genai";
@ -14,6 +15,7 @@ import type {
Api,
AssistantMessage,
Context,
ImageContent,
Model,
StopReason,
StreamFunction,
@ -238,7 +240,12 @@ export const streamGoogle: StreamFunction<"google-generative-ai"> = (
stream.push({ type: "done", reason: output.stopReason, message: output });
stream.end();
} catch (error) {
for (const block of output.content) delete (block as any).index;
// Remove internal index property used during streaming
for (const block of output.content) {
if ("index" in block) {
delete (block as { index?: number }).index;
}
}
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error);
stream.push({ type: "error", reason: output.stopReason, error: output });
@ -389,33 +396,33 @@ function convertMessages(model: Model<"google-generative-ai">, context: Context)
const parts: Part[] = [];
// Extract text and image content
const textResult = msg.content
.filter((c) => c.type === "text")
.map((c) => (c as any).text)
.join("\n");
const imageBlocks = model.input.includes("image") ? msg.content.filter((c) => c.type === "image") : [];
const textContent = msg.content.filter((c): c is TextContent => c.type === "text");
const textResult = textContent.map((c) => c.text).join("\n");
const imageContent = model.input.includes("image")
? msg.content.filter((c): c is ImageContent => c.type === "image")
: [];
// Always add functionResponse with text result (or placeholder if only images)
const hasText = textResult.length > 0;
const hasImages = imageBlocks.length > 0;
const hasImages = imageContent.length > 0;
// Use "output" key for success, "error" key for errors as per SDK documentation
const responseValue = hasText ? sanitizeSurrogates(textResult) : hasImages ? "(see attached image)" : "";
parts.push({
functionResponse: {
id: msg.toolCallId,
name: msg.toolName,
response: {
result: hasText ? sanitizeSurrogates(textResult) : hasImages ? "(see attached image)" : "",
isError: msg.isError,
},
response: msg.isError ? { error: responseValue } : { output: responseValue },
},
});
// Add any images as inlineData parts
for (const imageBlock of imageBlocks) {
for (const imageBlock of imageContent) {
parts.push({
inlineData: {
mimeType: (imageBlock as any).mimeType,
data: (imageBlock as any).data,
mimeType: imageBlock.mimeType,
data: imageBlock.data,
},
});
}
@ -430,14 +437,16 @@ function convertMessages(model: Model<"google-generative-ai">, context: Context)
return contents;
}
function convertTools(tools: Tool[]): any[] | undefined {
function convertTools(
tools: Tool[],
): { functionDeclarations: { name: string; description?: string; parameters: Schema }[] }[] | undefined {
if (tools.length === 0) return undefined;
return [
{
functionDeclarations: tools.map((tool) => ({
name: tool.name,
description: tool.description,
parameters: tool.parameters as any, // TypeBox already generates JSON Schema
parameters: tool.parameters as Schema, // TypeBox generates JSON Schema compatible with SDK Schema type
})),
},
];