From 39c626b6c9bfd1576d6dac6ac73f3ba8bd9b898e Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Tue, 16 Sep 2025 12:23:34 +0200 Subject: [PATCH] feat(ai): add partial JSON parsing for streaming tool calls - Added partial-json package for parsing incomplete JSON during streaming - Tool call arguments now contain partially parsed JSON during toolcall_delta events - Enables progressive UI updates (e.g., showing file paths before content is complete) - Arguments are always valid objects (minimum empty {}), never undefined - Full validation still occurs at toolcall_end when arguments are complete - Updated all providers (Anthropic, OpenAI Completions/Responses) to use parseStreamingJson - Added comprehensive documentation and examples in README - Added test to verify arguments are always defined during streaming --- package-lock.json | 11 +- packages/ai/README.md | 59 ++++++++-- packages/ai/package.json | 1 + packages/ai/src/json-parse.ts | 28 +++++ packages/ai/src/models.generated.ts | 108 +++++++++--------- packages/ai/src/providers/anthropic.ts | 2 + .../ai/src/providers/openai-completions.ts | 2 + packages/ai/src/providers/openai-responses.ts | 8 +- packages/ai/test/generate.test.ts | 6 + test-partial-json.js | 52 +++++++++ 10 files changed, 208 insertions(+), 69 deletions(-) create mode 100644 packages/ai/src/json-parse.ts create mode 100644 test-partial-json.js diff --git a/package-lock.json b/package-lock.json index 040d93eb..883227d9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1996,6 +1996,12 @@ } } }, + "node_modules/partial-json": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/partial-json/-/partial-json-0.1.7.tgz", + "integrity": "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA==", + "license": "MIT" + }, "node_modules/pathe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", @@ -2816,7 +2822,7 @@ "version": "0.5.39", "license": "MIT", "dependencies": { - "@mariozechner/pi-tui": "^0.5.38", + "@mariozechner/pi-tui": "^0.5.39", "@types/glob": "^8.1.0", "chalk": "^5.5.0", "glob": "^11.0.3", @@ -3205,6 +3211,7 @@ "ajv-formats": "^3.0.1", "chalk": "^5.6.2", "openai": "^5.20.0", + "partial-json": "^0.1.7", "zod-to-json-schema": "^3.24.6" }, "devDependencies": { @@ -3238,7 +3245,7 @@ "version": "0.5.39", "license": "MIT", "dependencies": { - "@mariozechner/pi-agent": "^0.5.38", + "@mariozechner/pi-agent": "^0.5.39", "chalk": "^5.5.0" }, "bin": { diff --git a/packages/ai/README.md b/packages/ai/README.md index f355c4a2..b913a689 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -194,6 +194,51 @@ for (const block of response.content) { } ``` +### Streaming Tool Calls with Partial JSON + +During streaming, tool call arguments are progressively parsed as they arrive. This enables real-time UI updates before the complete arguments are available: + +```typescript +const s = stream(model, context); + +for await (const event of s) { + if (event.type === 'toolcall_delta') { + const toolCall = event.partial.content[event.contentIndex]; + + // toolCall.arguments contains partially parsed JSON during streaming + // This allows for progressive UI updates + if (toolCall.type === 'toolCall' && toolCall.arguments) { + // BE DEFENSIVE: arguments may be incomplete + // Example: Show file path being written even before content is complete + if (toolCall.name === 'write_file' && toolCall.arguments.path) { + console.log(`Writing to: ${toolCall.arguments.path}`); + + // Content might be partial or missing + if (toolCall.arguments.content) { + console.log(`Content preview: ${toolCall.arguments.content.substring(0, 100)}...`); + } + } + } + } + + if (event.type === 'toolcall_end') { + // Here toolCall.arguments is complete and validated + const toolCall = event.toolCall; + console.log(`Tool completed: ${toolCall.name}`, toolCall.arguments); + } +} +``` + +**Important notes about partial tool arguments:** +- During `toolcall_delta` events, `arguments` contains the best-effort parse of partial JSON +- Fields may be missing or incomplete - always check for existence before use +- String values may be truncated mid-word +- Arrays may be incomplete +- Nested objects may be partially populated +- At minimum, `arguments` will be an empty object `{}`, never `undefined` +- Full validation only occurs at `toolcall_end` when arguments are complete +- The Google provider does not support function call streaming. Instead, you will receive a single `toolcall_delta` even with the full arguments. + ## Image Input Models with vision capabilities can process images. You can check if a model supports images via the `input` property. If you pass images to a non-vision model, they are silently ignored. @@ -642,26 +687,26 @@ for await (const event of stream) { case 'agent_start': console.log('Agent started'); break; - + case 'turn_start': console.log('New turn started'); break; - + case 'message_start': console.log(`${event.message.role} message started`); break; - + case 'message_update': // Only for assistant messages during streaming if (event.message.content.some(c => c.type === 'text')) { console.log('Assistant:', event.message.content); } break; - + case 'tool_execution_start': console.log(`Calling ${event.toolName} with:`, event.args); break; - + case 'tool_execution_end': if (event.isError) { console.error(`Tool failed:`, event.result); @@ -669,11 +714,11 @@ for await (const event of stream) { console.log(`Tool result:`, event.result.output); } break; - + case 'turn_end': console.log(`Turn ended with ${event.toolResults.length} tool calls`); break; - + case 'agent_end': console.log(`Agent completed with ${event.messages.length} new messages`); break; diff --git a/packages/ai/package.json b/packages/ai/package.json index 0f21af44..71b0c33d 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -26,6 +26,7 @@ "ajv-formats": "^3.0.1", "chalk": "^5.6.2", "openai": "^5.20.0", + "partial-json": "^0.1.7", "zod-to-json-schema": "^3.24.6" }, "keywords": [ diff --git a/packages/ai/src/json-parse.ts b/packages/ai/src/json-parse.ts new file mode 100644 index 00000000..feeb32ad --- /dev/null +++ b/packages/ai/src/json-parse.ts @@ -0,0 +1,28 @@ +import { parse as partialParse } from "partial-json"; + +/** + * Attempts to parse potentially incomplete JSON during streaming. + * Always returns a valid object, even if the JSON is incomplete. + * + * @param partialJson The partial JSON string from streaming + * @returns Parsed object or empty object if parsing fails + */ +export function parseStreamingJson(partialJson: string | undefined): T { + if (!partialJson || partialJson.trim() === "") { + return {} as T; + } + + // Try standard parsing first (fastest for complete JSON) + try { + return JSON.parse(partialJson) as T; + } catch { + // Try partial-json for incomplete JSON + try { + const result = partialParse(partialJson); + return (result ?? {}) as T; + } catch { + // If all parsing fails, return empty object + return {} as T; + } + } +} diff --git a/packages/ai/src/models.generated.ts b/packages/ai/src/models.generated.ts index 88be0a5d..f4626e87 100644 --- a/packages/ai/src/models.generated.ts +++ b/packages/ai/src/models.generated.ts @@ -2714,13 +2714,13 @@ export const MODELS = { reasoning: false, input: ["text"], cost: { - input: 0.038000000000000006, - output: 0.12, + input: 0.012, + output: 0.036, cacheRead: 0, cacheWrite: 0, }, contextWindow: 131072, - maxTokens: 16384, + maxTokens: 8192, } satisfies Model<"openai-completions">, "amazon/nova-lite-v1": { id: "amazon/nova-lite-v1", @@ -2943,23 +2943,6 @@ export const MODELS = { contextWindow: 32768, maxTokens: 4096, } satisfies Model<"openai-completions">, - "cohere/command-r-plus-08-2024": { - id: "cohere/command-r-plus-08-2024", - name: "Cohere: Command R+ (08-2024)", - api: "openai-completions", - provider: "openrouter", - baseUrl: "https://openrouter.ai/api/v1", - reasoning: false, - input: ["text"], - cost: { - input: 2.5, - output: 10, - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: 128000, - maxTokens: 4000, - } satisfies Model<"openai-completions">, "cohere/command-r-08-2024": { id: "cohere/command-r-08-2024", name: "Cohere: Command R (08-2024)", @@ -2977,6 +2960,23 @@ export const MODELS = { contextWindow: 128000, maxTokens: 4000, } satisfies Model<"openai-completions">, + "cohere/command-r-plus-08-2024": { + id: "cohere/command-r-plus-08-2024", + name: "Cohere: Command R+ (08-2024)", + api: "openai-completions", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + reasoning: false, + input: ["text"], + cost: { + input: 2.5, + output: 10, + cacheRead: 0, + cacheWrite: 0, + }, + contextWindow: 128000, + maxTokens: 4000, + } satisfies Model<"openai-completions">, "microsoft/phi-3.5-mini-128k-instruct": { id: "microsoft/phi-3.5-mini-128k-instruct", name: "Microsoft: Phi-3.5 Mini 128K Instruct", @@ -3079,23 +3079,6 @@ export const MODELS = { contextWindow: 131072, maxTokens: 128000, } satisfies Model<"openai-completions">, - "mistralai/mistral-7b-instruct-v0.3": { - id: "mistralai/mistral-7b-instruct-v0.3", - name: "Mistral: Mistral 7B Instruct v0.3", - api: "openai-completions", - provider: "openrouter", - baseUrl: "https://openrouter.ai/api/v1", - reasoning: false, - input: ["text"], - cost: { - input: 0.028, - output: 0.054, - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: 32768, - maxTokens: 16384, - } satisfies Model<"openai-completions">, "mistralai/mistral-7b-instruct:free": { id: "mistralai/mistral-7b-instruct:free", name: "Mistral: Mistral 7B Instruct (free)", @@ -3130,6 +3113,23 @@ export const MODELS = { contextWindow: 32768, maxTokens: 16384, } satisfies Model<"openai-completions">, + "mistralai/mistral-7b-instruct-v0.3": { + id: "mistralai/mistral-7b-instruct-v0.3", + name: "Mistral: Mistral 7B Instruct v0.3", + api: "openai-completions", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + reasoning: false, + input: ["text"], + cost: { + input: 0.028, + output: 0.054, + cacheRead: 0, + cacheWrite: 0, + }, + contextWindow: 32768, + maxTokens: 16384, + } satisfies Model<"openai-completions">, "microsoft/phi-3-mini-128k-instruct": { id: "microsoft/phi-3-mini-128k-instruct", name: "Microsoft: Phi-3 Mini 128K Instruct", @@ -3300,23 +3300,6 @@ export const MODELS = { contextWindow: 128000, maxTokens: 4096, } satisfies Model<"openai-completions">, - "mistralai/mistral-tiny": { - id: "mistralai/mistral-tiny", - name: "Mistral Tiny", - api: "openai-completions", - provider: "openrouter", - baseUrl: "https://openrouter.ai/api/v1", - reasoning: false, - input: ["text"], - cost: { - input: 0.25, - output: 0.25, - cacheRead: 0, - cacheWrite: 0, - }, - contextWindow: 32768, - maxTokens: 4096, - } satisfies Model<"openai-completions">, "mistralai/mistral-small": { id: "mistralai/mistral-small", name: "Mistral Small", @@ -3334,6 +3317,23 @@ export const MODELS = { contextWindow: 32768, maxTokens: 4096, } satisfies Model<"openai-completions">, + "mistralai/mistral-tiny": { + id: "mistralai/mistral-tiny", + name: "Mistral Tiny", + api: "openai-completions", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + reasoning: false, + input: ["text"], + cost: { + input: 0.25, + output: 0.25, + cacheRead: 0, + cacheWrite: 0, + }, + contextWindow: 32768, + maxTokens: 4096, + } satisfies Model<"openai-completions">, "mistralai/mixtral-8x7b-instruct": { id: "mistralai/mixtral-8x7b-instruct", name: "Mistral: Mixtral 8x7B Instruct", diff --git a/packages/ai/src/providers/anthropic.ts b/packages/ai/src/providers/anthropic.ts index 7a7f60ef..57cea2ac 100644 --- a/packages/ai/src/providers/anthropic.ts +++ b/packages/ai/src/providers/anthropic.ts @@ -5,6 +5,7 @@ import type { MessageParam, } from "@anthropic-ai/sdk/resources/messages.js"; import { AssistantMessageEventStream } from "../event-stream.js"; +import { parseStreamingJson } from "../json-parse.js"; import { calculateCost } from "../models.js"; import type { Api, @@ -124,6 +125,7 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = ( const block = blocks[index]; if (block && block.type === "toolCall") { block.partialJson += event.delta.partial_json; + block.arguments = parseStreamingJson(block.partialJson); stream.push({ type: "toolcall_delta", contentIndex: index, diff --git a/packages/ai/src/providers/openai-completions.ts b/packages/ai/src/providers/openai-completions.ts index 02b6c26a..11398b5a 100644 --- a/packages/ai/src/providers/openai-completions.ts +++ b/packages/ai/src/providers/openai-completions.ts @@ -8,6 +8,7 @@ import type { ChatCompletionMessageParam, } from "openai/resources/chat/completions.js"; import { AssistantMessageEventStream } from "../event-stream.js"; +import { parseStreamingJson } from "../json-parse.js"; import { calculateCost } from "../models.js"; import type { AssistantMessage, @@ -210,6 +211,7 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions"> = ( if (toolCall.function?.arguments) { delta = toolCall.function.arguments; currentBlock.partialArgs += toolCall.function.arguments; + currentBlock.arguments = parseStreamingJson(currentBlock.partialArgs); } stream.push({ type: "toolcall_delta", diff --git a/packages/ai/src/providers/openai-responses.ts b/packages/ai/src/providers/openai-responses.ts index e3077494..8cf05228 100644 --- a/packages/ai/src/providers/openai-responses.ts +++ b/packages/ai/src/providers/openai-responses.ts @@ -11,6 +11,7 @@ import type { ResponseReasoningItem, } from "openai/resources/responses/responses.js"; import { AssistantMessageEventStream } from "../event-stream.js"; +import { parseStreamingJson } from "../json-parse.js"; import { calculateCost } from "../models.js"; import type { Api, @@ -194,12 +195,7 @@ export const streamOpenAIResponses: StreamFunction<"openai-responses"> = ( currentBlock.type === "toolCall" ) { currentBlock.partialJson += event.delta; - try { - const args = JSON.parse(currentBlock.partialJson); - currentBlock.arguments = args; - } catch { - // Ignore JSON parse errors - the JSON might be incomplete - } + currentBlock.arguments = parseStreamingJson(currentBlock.partialJson); stream.push({ type: "toolcall_delta", contentIndex: blockIndex(), diff --git a/packages/ai/test/generate.test.ts b/packages/ai/test/generate.test.ts index a62918a3..90c16b99 100644 --- a/packages/ai/test/generate.test.ts +++ b/packages/ai/test/generate.test.ts @@ -95,6 +95,12 @@ async function handleToolCall(model: Model, options?: Op if (toolCall.type === "toolCall") { expect(toolCall.name).toBe("calculator"); accumulatedToolArgs += event.delta; + // Check that we have a parsed arguments object during streaming + expect(toolCall.arguments).toBeDefined(); + expect(typeof toolCall.arguments).toBe("object"); + // The arguments should be partially populated as we stream + // At minimum it should be an empty object, never undefined + expect(toolCall.arguments).not.toBeNull(); } } if (event.type === "toolcall_end") { diff --git a/test-partial-json.js b/test-partial-json.js new file mode 100644 index 00000000..92e00b7e --- /dev/null +++ b/test-partial-json.js @@ -0,0 +1,52 @@ +import { parseStreamingJson } from "./packages/ai/dist/json-parse.js"; + +// Test cases for partial JSON parsing +const testCases = [ + // Complete JSON + { input: '{"name":"test","value":42}', expected: {name: "test", value: 42} }, + + // Partial JSON - incomplete object + { input: '{"name":"test","val', expected: {name: "test"} }, + { input: '{"name":"test"', expected: {name: "test"} }, + { input: '{"name":', expected: {} }, + { input: '{"', expected: {} }, + { input: '{', expected: {} }, + + // Partial JSON - incomplete array + { input: '{"items":[1,2,3', expected: {items: [1, 2, 3]} }, + { input: '{"items":[1,2,', expected: {items: [1, 2]} }, + { input: '{"items":[', expected: {items: []} }, + + // Partial JSON - incomplete string + { input: '{"message":"Hello wor', expected: {message: "Hello wor"} }, + + // Empty or invalid + { input: '', expected: {} }, + { input: null, expected: {} }, + { input: undefined, expected: {} }, + + // Complex nested partial + { input: '{"user":{"name":"John","age":30,"address":{"city":"New Y', expected: {user: {name: "John", age: 30, address: {city: "New Y"}}} }, +]; + +console.log("Testing parseStreamingJson...\n"); + +let passed = 0; +let failed = 0; + +for (const test of testCases) { + const result = parseStreamingJson(test.input); + const success = JSON.stringify(result) === JSON.stringify(test.expected); + + if (success) { + console.log(`✅ PASS: "${test.input || '(empty)'}" -> ${JSON.stringify(result)}`); + passed++; + } else { + console.log(`❌ FAIL: "${test.input || '(empty)'}"`); + console.log(` Expected: ${JSON.stringify(test.expected)}`); + console.log(` Got: ${JSON.stringify(result)}`); + failed++; + } +} + +console.log(`\n${passed} passed, ${failed} failed`); \ No newline at end of file