diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index df39e925..9897984d 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -221,7 +221,6 @@ export class Agent { tools: this._state.tools, model, reasoning, - validateToolCallsAtProvider: false, getQueuedMessages: async () => { // Return queued messages based on queue mode if (this.queueMode === "one-at-a-time") { diff --git a/packages/agent/src/transports/AppTransport.ts b/packages/agent/src/transports/AppTransport.ts index 0404f37a..5beb9dc6 100644 --- a/packages/agent/src/transports/AppTransport.ts +++ b/packages/agent/src/transports/AppTransport.ts @@ -77,7 +77,6 @@ function streamSimpleProxy( temperature: options.temperature, maxTokens: options.maxTokens, reasoning: options.reasoning, - validateToolCallsAtProvider: options.validateToolCallsAtProvider, // Don't send apiKey or signal - those are added server-side }, }), @@ -366,7 +365,6 @@ export class AppTransport implements AgentTransport { model: cfg.model, reasoning: cfg.reasoning, getQueuedMessages: cfg.getQueuedMessages, - validateToolCallsAtProvider: cfg.validateToolCallsAtProvider ?? false, }; // Yield events from the upstream agentLoop iterator diff --git a/packages/agent/src/transports/ProviderTransport.ts b/packages/agent/src/transports/ProviderTransport.ts index 1435b160..c46b16c0 100644 --- a/packages/agent/src/transports/ProviderTransport.ts +++ b/packages/agent/src/transports/ProviderTransport.ts @@ -65,7 +65,6 @@ export class ProviderTransport implements AgentTransport { reasoning: cfg.reasoning, apiKey, getQueuedMessages: cfg.getQueuedMessages, - validateToolCallsAtProvider: cfg.validateToolCallsAtProvider ?? false, }; // Yield events from agentLoop diff --git a/packages/agent/src/transports/types.ts b/packages/agent/src/transports/types.ts index 43982a69..d5d4053c 100644 --- a/packages/agent/src/transports/types.ts +++ b/packages/agent/src/transports/types.ts @@ -9,7 +9,6 @@ export interface AgentRunConfig { model: Model; reasoning?: "low" | "medium" | "high"; getQueuedMessages?: () => Promise[]>; - validateToolCallsAtProvider?: boolean; } /** diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 9cd5b522..a333e15b 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -2,9 +2,13 @@ ## [Unreleased] +### Breaking Changes + +- Removed provider-level tool argument validation. Validation now happens in `agentLoop` via `executeToolCalls`, allowing models to retry on validation errors. For manual tool execution, use `validateToolCall(tools, toolCall)` or `validateToolArguments(tool, toolCall)`. + ### Added -- Added `validateToolCallsAtProvider` option to streaming and agent APIs to optionally skip provider-level tool-call validation (default on), allowing agent loops to surface schema errors as toolResult messages and retry. +- Added `validateToolCall(tools, toolCall)` helper that finds the tool by name and validates arguments. ## [0.13.0] - 2025-12-06 diff --git a/packages/ai/README.md b/packages/ai/README.md index acd4b78e..52d69605 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -194,8 +194,8 @@ const response = await complete(model, context); // Check for tool calls in the response for (const block of response.content) { if (block.type === 'toolCall') { - // Arguments are automatically validated against the TypeBox schema using AJV - // If validation fails, an error event is emitted + // Execute your tool with the arguments + // See "Validating Tool Arguments" section for validation const result = await executeWeatherApi(block.arguments); // Add tool result with text content @@ -253,7 +253,7 @@ for await (const event of s) { } if (event.type === 'toolcall_end') { - // Here toolCall.arguments is complete and validated + // Here toolCall.arguments is complete (but not yet validated) const toolCall = event.toolCall; console.log(`Tool completed: ${toolCall.name}`, toolCall.arguments); } @@ -267,22 +267,43 @@ for await (const event of s) { - Arrays may be incomplete - Nested objects may be partially populated - At minimum, `arguments` will be an empty object `{}`, never `undefined` -- Full validation only occurs at `toolcall_end` when arguments are complete - The Google provider does not support function call streaming. Instead, you will receive a single `toolcall_delta` event with the full arguments. -### Provider tool-call validation +### Validating Tool Arguments -By default, providers validate streamed tool calls against your tool schema and abort the stream on validation errors. Set `validateToolCallsAtProvider: false` on `stream`, `streamSimple`, `complete`, `completeSimple`, or `AgentLoopConfig` to skip provider-level validation and let downstream code (for example, `agentLoop` via `executeToolCalls` → `validateToolArguments`) surface schema errors as `toolResult` messages. This enables the model to retry after receiving a validation error. +When using `agentLoop`, tool arguments are automatically validated against your TypeBox schemas before execution. If validation fails, the error is returned to the model as a tool result, allowing it to retry. + +When implementing your own tool execution loop with `stream()` or `complete()`, use `validateToolCall` to validate arguments before passing them to your tools: ```typescript -await streamSimple(model, context, { - apiKey: 'your-key', - validateToolCallsAtProvider: false -}); -``` +import { stream, validateToolCall, Tool } from '@mariozechner/pi-ai'; -- `true` (default): Provider validates tool calls and emits an error if arguments do not match the schema -- `false`: Provider emits tool calls even when arguments are invalid; callers must validate and handle errors themselves +const tools: Tool[] = [weatherTool, calculatorTool]; +const s = stream(model, { messages, tools }); + +for await (const event of s) { + if (event.type === 'toolcall_end') { + const toolCall = event.toolCall; + + try { + // Validate arguments against the tool's schema (throws on invalid args) + const validatedArgs = validateToolCall(tools, toolCall); + const result = await executeMyTool(toolCall.name, validatedArgs); + // ... add tool result to context + } catch (error) { + // Validation failed - return error as tool result so model can retry + context.messages.push({ + role: 'toolResult', + toolCallId: toolCall.id, + toolName: toolCall.name, + content: [{ type: 'text', text: error.message }], + isError: true, + timestamp: Date.now() + }); + } + } +} +``` ### Complete Event Reference diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 874686b7..f9ed0753 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -8,3 +8,4 @@ export * from "./stream.js"; export * from "./types.js"; export * from "./utils/overflow.js"; export * from "./utils/typebox-helpers.js"; +export * from "./utils/validation.js"; diff --git a/packages/ai/src/providers/anthropic.ts b/packages/ai/src/providers/anthropic.ts index d3421553..ff6e60e2 100644 --- a/packages/ai/src/providers/anthropic.ts +++ b/packages/ai/src/providers/anthropic.ts @@ -25,7 +25,7 @@ import type { import { AssistantMessageEventStream } from "../utils/event-stream.js"; import { parseStreamingJson } from "../utils/json-parse.js"; import { sanitizeSurrogates } from "../utils/sanitize-unicode.js"; -import { validateToolArguments } from "../utils/validation.js"; + import { transformMessages } from "./transorm-messages.js"; /** @@ -92,7 +92,6 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = ( options?: AnthropicOptions, ): AssistantMessageEventStream => { const stream = new AssistantMessageEventStream(); - const shouldValidateToolCalls = options?.validateToolCallsAtProvider !== false; (async () => { const output: AssistantMessage = { @@ -232,15 +231,6 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = ( }); } else if (block.type === "toolCall") { block.arguments = parseStreamingJson(block.partialJson); - - // Validate tool arguments if tool definition is available - if (shouldValidateToolCalls && context.tools) { - const tool = context.tools.find((t) => t.name === block.name); - if (tool) { - block.arguments = validateToolArguments(tool, block); - } - } - delete (block as any).partialJson; stream.push({ type: "toolcall_end", diff --git a/packages/ai/src/providers/google.ts b/packages/ai/src/providers/google.ts index e571aeb0..5b5a0356 100644 --- a/packages/ai/src/providers/google.ts +++ b/packages/ai/src/providers/google.ts @@ -23,7 +23,7 @@ import type { } from "../types.js"; import { AssistantMessageEventStream } from "../utils/event-stream.js"; import { sanitizeSurrogates } from "../utils/sanitize-unicode.js"; -import { validateToolArguments } from "../utils/validation.js"; + import { transformMessages } from "./transorm-messages.js"; export interface GoogleOptions extends StreamOptions { @@ -43,7 +43,6 @@ export const streamGoogle: StreamFunction<"google-generative-ai"> = ( options?: GoogleOptions, ): AssistantMessageEventStream => { const stream = new AssistantMessageEventStream(); - const shouldValidateToolCalls = options?.validateToolCallsAtProvider !== false; (async () => { const output: AssistantMessage = { @@ -167,14 +166,6 @@ export const streamGoogle: StreamFunction<"google-generative-ai"> = ( ...(part.thoughtSignature && { thoughtSignature: part.thoughtSignature }), }; - // Validate tool arguments if tool definition is available - if (shouldValidateToolCalls && context.tools) { - const tool = context.tools.find((t) => t.name === toolCall.name); - if (tool) { - toolCall.arguments = validateToolArguments(tool, toolCall); - } - } - output.content.push(toolCall); stream.push({ type: "toolcall_start", contentIndex: blockIndex(), partial: output }); stream.push({ diff --git a/packages/ai/src/providers/openai-completions.ts b/packages/ai/src/providers/openai-completions.ts index dafacd9e..ca9f1c30 100644 --- a/packages/ai/src/providers/openai-completions.ts +++ b/packages/ai/src/providers/openai-completions.ts @@ -23,7 +23,7 @@ import type { import { AssistantMessageEventStream } from "../utils/event-stream.js"; import { parseStreamingJson } from "../utils/json-parse.js"; import { sanitizeSurrogates } from "../utils/sanitize-unicode.js"; -import { validateToolArguments } from "../utils/validation.js"; + import { transformMessages } from "./transorm-messages.js"; export interface OpenAICompletionsOptions extends StreamOptions { @@ -37,7 +37,6 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions"> = ( options?: OpenAICompletionsOptions, ): AssistantMessageEventStream => { const stream = new AssistantMessageEventStream(); - const shouldValidateToolCalls = options?.validateToolCallsAtProvider !== false; (async () => { const output: AssistantMessage = { @@ -85,15 +84,6 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions"> = ( }); } else if (block.type === "toolCall") { block.arguments = JSON.parse(block.partialArgs || "{}"); - - // Validate tool arguments if tool definition is available - if (shouldValidateToolCalls && context.tools) { - const tool = context.tools.find((t) => t.name === block.name); - if (tool) { - block.arguments = validateToolArguments(tool, block); - } - } - delete block.partialArgs; stream.push({ type: "toolcall_end", diff --git a/packages/ai/src/providers/openai-responses.ts b/packages/ai/src/providers/openai-responses.ts index 9b2c4fb6..c36e5254 100644 --- a/packages/ai/src/providers/openai-responses.ts +++ b/packages/ai/src/providers/openai-responses.ts @@ -27,7 +27,7 @@ import type { import { AssistantMessageEventStream } from "../utils/event-stream.js"; import { parseStreamingJson } from "../utils/json-parse.js"; import { sanitizeSurrogates } from "../utils/sanitize-unicode.js"; -import { validateToolArguments } from "../utils/validation.js"; + import { transformMessages } from "./transorm-messages.js"; // OpenAI Responses-specific options @@ -45,7 +45,6 @@ export const streamOpenAIResponses: StreamFunction<"openai-responses"> = ( options?: OpenAIResponsesOptions, ): AssistantMessageEventStream => { const stream = new AssistantMessageEventStream(); - const shouldValidateToolCalls = options?.validateToolCallsAtProvider !== false; // Start async processing (async () => { @@ -240,14 +239,6 @@ export const streamOpenAIResponses: StreamFunction<"openai-responses"> = ( arguments: JSON.parse(item.arguments), }; - // Validate tool arguments if tool definition is available - if (shouldValidateToolCalls && context.tools) { - const tool = context.tools.find((t) => t.name === toolCall.name); - if (tool) { - toolCall.arguments = validateToolArguments(tool, toolCall); - } - } - stream.push({ type: "toolcall_end", contentIndex: blockIndex(), toolCall, partial: output }); } } diff --git a/packages/ai/src/stream.ts b/packages/ai/src/stream.ts index 9c5b20d7..94fb21b1 100644 --- a/packages/ai/src/stream.ts +++ b/packages/ai/src/stream.ts @@ -120,7 +120,6 @@ function mapOptionsForApi( maxTokens: options?.maxTokens || Math.min(model.maxTokens, 32000), signal: options?.signal, apiKey: apiKey || options?.apiKey, - validateToolCallsAtProvider: options?.validateToolCallsAtProvider ?? true, }; switch (model.api) { diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index b3b8a885..a7269bc8 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -37,12 +37,6 @@ export interface StreamOptions { maxTokens?: number; signal?: AbortSignal; apiKey?: string; - /** - * Controls whether providers validate streamed tool calls against the tool schema. - * Defaults to true. Set to false to skip provider-level validation and allow - * downstream consumers (e.g., agentLoop) to handle validation failures. - */ - validateToolCallsAtProvider?: boolean; } // Unified options with reasoning passed to streamSimple() and completeSimple() diff --git a/packages/ai/src/utils/validation.ts b/packages/ai/src/utils/validation.ts index 08335807..4c778880 100644 --- a/packages/ai/src/utils/validation.ts +++ b/packages/ai/src/utils/validation.ts @@ -27,6 +27,21 @@ if (!isBrowserExtension) { } } +/** + * Finds a tool by name and validates the tool call arguments against its TypeBox schema + * @param tools Array of tool definitions + * @param toolCall The tool call from the LLM + * @returns The validated arguments + * @throws Error if tool is not found or validation fails + */ +export function validateToolCall(tools: Tool[], toolCall: ToolCall): any { + const tool = tools.find((t) => t.name === toolCall.name); + if (!tool) { + throw new Error(`Tool "${toolCall.name}" not found`); + } + return validateToolArguments(tool, toolCall); +} + /** * Validates tool call arguments against the tool's TypeBox schema * @param tool The tool definition with TypeBox schema