From 19f3c23f6d9ea633fa3a26ee8bd20fe4cade8674 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Tue, 13 Jan 2026 16:46:00 +0100 Subject: [PATCH] Fix PR #689: Add changelog attribution, coding-agent changelog, fix test types, add provider to test suites - Fix ai/CHANGELOG.md: add PR link and author attribution - Add coding-agent/CHANGELOG.md entry for vercel-ai-gateway provider - Fix model-resolver.test.ts: use anthropic-messages API type to match generated models - Add vercel-ai-gateway to test suites: tokens, abort, empty, context-overflow, unicode-surrogate, tool-call-without-result, image-tool-result, total-tokens, image-limits --- packages/ai/CHANGELOG.md | 2 +- packages/ai/test/abort.test.ts | 12 ++++++ packages/ai/test/context-overflow.test.ts | 15 +++++++ packages/ai/test/empty.test.ts | 20 +++++++++ packages/ai/test/image-limits.test.ts | 43 +++++++++++++++++++ packages/ai/test/image-tool-result.test.ts | 12 ++++++ packages/ai/test/tokens.test.ts | 8 ++++ .../ai/test/tool-call-without-result.test.ts | 8 ++++ packages/ai/test/total-tokens.test.ts | 23 ++++++++++ packages/ai/test/unicode-surrogate.test.ts | 16 +++++++ packages/coding-agent/CHANGELOG.md | 1 + .../coding-agent/test/model-resolver.test.ts | 6 +-- 12 files changed, 162 insertions(+), 4 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 2ac0ff3c..02efc9c3 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -4,7 +4,7 @@ ### Added -- Added Vercel AI Gateway provider with model discovery and `AI_GATEWAY_API_KEY` env support. +- Added Vercel AI Gateway provider with model discovery and `AI_GATEWAY_API_KEY` env support ([#689](https://github.com/badlogic/pi-mono/pull/689) by [@timolins](https://github.com/timolins)) ## [0.45.3] - 2026-01-13 diff --git a/packages/ai/test/abort.test.ts b/packages/ai/test/abort.test.ts index a0594edd..880f638d 100644 --- a/packages/ai/test/abort.test.ts +++ b/packages/ai/test/abort.test.ts @@ -172,6 +172,18 @@ describe("AI Providers Abort Tests", () => { }); }); + describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider Abort", () => { + const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash"); + + it("should abort mid-stream", { retry: 3 }, async () => { + await testAbortSignal(llm); + }); + + it("should handle immediate abort", { retry: 3 }, async () => { + await testImmediateAbort(llm); + }); + }); + // Google Gemini CLI / Antigravity share the same provider, so one test covers both describe("Google Gemini CLI Provider Abort", () => { it.skipIf(!geminiCliToken)("should abort mid-stream", { retry: 3 }, async () => { diff --git a/packages/ai/test/context-overflow.test.ts b/packages/ai/test/context-overflow.test.ts index eb40b893..8ccf0cfb 100644 --- a/packages/ai/test/context-overflow.test.ts +++ b/packages/ai/test/context-overflow.test.ts @@ -412,6 +412,21 @@ describe("Context overflow error handling", () => { }, 120000); }); + // ============================================================================= + // Vercel AI Gateway - Unified API for multiple providers + // ============================================================================= + + describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway", () => { + it("google/gemini-2.5-flash via AI Gateway - should detect overflow via isContextOverflow", async () => { + const model = getModel("vercel-ai-gateway", "google/gemini-2.5-flash"); + const result = await testContextOverflow(model, process.env.AI_GATEWAY_API_KEY!); + logResult(result); + + expect(result.stopReason).toBe("error"); + expect(isContextOverflow(result.response, model.contextWindow)).toBe(true); + }, 120000); + }); + // ============================================================================= // OpenRouter - Multiple backend providers // Expected pattern: "maximum context length is X tokens" diff --git a/packages/ai/test/empty.test.ts b/packages/ai/test/empty.test.ts index b19aa8a1..12415f6c 100644 --- a/packages/ai/test/empty.test.ts +++ b/packages/ai/test/empty.test.ts @@ -342,6 +342,26 @@ describe("AI Providers Empty Message Tests", () => { }); }); + describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider Empty Messages", () => { + const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash"); + + it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => { + await testEmptyMessage(llm); + }); + + it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => { + await testEmptyStringMessage(llm); + }); + + it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => { + await testWhitespaceOnlyMessage(llm); + }); + + it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => { + await testEmptyAssistantMessage(llm); + }); + }); + describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider Empty Messages", () => { const llm = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0"); diff --git a/packages/ai/test/image-limits.test.ts b/packages/ai/test/image-limits.test.ts index 17fdfaf3..a4d6e8df 100644 --- a/packages/ai/test/image-limits.test.ts +++ b/packages/ai/test/image-limits.test.ts @@ -841,6 +841,49 @@ describe("Image Limits E2E Tests", () => { }); }); + // ------------------------------------------------------------------------- + // Vercel AI Gateway (google/gemini-2.5-flash) + // ------------------------------------------------------------------------- + describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway (google/gemini-2.5-flash)", () => { + const model = getModel("vercel-ai-gateway", "google/gemini-2.5-flash"); + + it("should accept a small number of images (5)", async () => { + const result = await testImageCount(model, 5, smallImage); + expect(result.success, result.error).toBe(true); + }); + + it("should find maximum image count limit", { timeout: 600000 }, async () => { + const { limit, lastError } = await findLimit((count) => testImageCount(model, count, smallImage), 10, 100, 10); + console.log(`\n Vercel AI Gateway max images: ~${limit} (last error: ${lastError})`); + expect(limit).toBeGreaterThanOrEqual(5); + }); + + it("should find maximum image size limit", { timeout: 600000 }, async () => { + const MB = 1024 * 1024; + const sizes = [5, 10, 15, 20]; + + let lastSuccess = 0; + let lastError: string | undefined; + + for (const sizeMB of sizes) { + console.log(` Testing size: ${sizeMB}MB...`); + const imageBase64 = generateImageWithSize(sizeMB * MB, `size-${sizeMB}mb.png`); + const result = await testImageSize(model, imageBase64); + if (result.success) { + lastSuccess = sizeMB; + console.log(` SUCCESS`); + } else { + lastError = result.error; + console.log(` FAILED: ${result.error?.substring(0, 100)}`); + break; + } + } + + console.log(`\n Vercel AI Gateway max image size: ~${lastSuccess}MB (last error: ${lastError})`); + expect(lastSuccess).toBeGreaterThanOrEqual(5); + }); + }); + // ------------------------------------------------------------------------- // Amazon Bedrock (claude-sonnet-4-5) // Limits: 100 images (Anthropic), 5MB per image, 8000px max dimension diff --git a/packages/ai/test/image-tool-result.test.ts b/packages/ai/test/image-tool-result.test.ts index a757be2c..c1981802 100644 --- a/packages/ai/test/image-tool-result.test.ts +++ b/packages/ai/test/image-tool-result.test.ts @@ -274,6 +274,18 @@ describe("Tool Results with Images", () => { }); }); + describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider (google/gemini-2.5-flash)", () => { + const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash"); + + it("should handle tool result with only image", { retry: 3, timeout: 30000 }, async () => { + await handleToolWithImageResult(llm); + }); + + it("should handle tool result with text and image", { retry: 3, timeout: 30000 }, async () => { + await handleToolWithTextAndImageResult(llm); + }); + }); + describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider (claude-sonnet-4-5)", () => { const llm = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0"); diff --git a/packages/ai/test/tokens.test.ts b/packages/ai/test/tokens.test.ts index 0297a8fe..6d10abcf 100644 --- a/packages/ai/test/tokens.test.ts +++ b/packages/ai/test/tokens.test.ts @@ -159,6 +159,14 @@ describe("Token Statistics on Abort", () => { }); }); + describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider", () => { + const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash"); + + it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => { + await testTokensOnAbort(llm); + }); + }); + // ========================================================================= // OAuth-based providers (credentials from ~/.pi/agent/oauth.json) // ========================================================================= diff --git a/packages/ai/test/tool-call-without-result.test.ts b/packages/ai/test/tool-call-without-result.test.ts index c8e33a53..0fe6f4bf 100644 --- a/packages/ai/test/tool-call-without-result.test.ts +++ b/packages/ai/test/tool-call-without-result.test.ts @@ -179,6 +179,14 @@ describe("Tool Call Without Result Tests", () => { }); }); + describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider", () => { + const model = getModel("vercel-ai-gateway", "google/gemini-2.5-flash"); + + it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => { + await testToolCallWithoutResult(model); + }); + }); + describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider", () => { const model = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0"); diff --git a/packages/ai/test/total-tokens.test.ts b/packages/ai/test/total-tokens.test.ts index ee89af11..3b394401 100644 --- a/packages/ai/test/total-tokens.test.ts +++ b/packages/ai/test/total-tokens.test.ts @@ -348,6 +348,29 @@ describe("totalTokens field", () => { ); }); + // ========================================================================= + // Vercel AI Gateway + // ========================================================================= + + describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway", () => { + it( + "google/gemini-2.5-flash - should return totalTokens equal to sum of components", + { retry: 3, timeout: 60000 }, + async () => { + const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash"); + + console.log(`\nVercel AI Gateway / ${llm.id}:`); + const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.AI_GATEWAY_API_KEY }); + + logUsage("First request", first); + logUsage("Second request", second); + + assertTotalTokensEqualsComponents(first); + assertTotalTokensEqualsComponents(second); + }, + ); + }); + // ========================================================================= // OpenRouter - Multiple backend providers // ========================================================================= diff --git a/packages/ai/test/unicode-surrogate.test.ts b/packages/ai/test/unicode-surrogate.test.ts index b28c41d3..7397034c 100644 --- a/packages/ai/test/unicode-surrogate.test.ts +++ b/packages/ai/test/unicode-surrogate.test.ts @@ -634,6 +634,22 @@ describe("AI Providers Unicode Surrogate Pair Tests", () => { }); }); + describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider Unicode Handling", () => { + const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash"); + + it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => { + await testEmojiInToolResults(llm); + }); + + it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => { + await testRealWorldLinkedInData(llm); + }); + + it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => { + await testUnpairedHighSurrogate(llm); + }); + }); + describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider Unicode Handling", () => { const llm = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0"); diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 07a1f3f6..545939af 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -9,6 +9,7 @@ ### Added - Extension example: `summarize.ts` for summarizing conversations using custom UI and an external model +- Vercel AI Gateway provider support: set `AI_GATEWAY_API_KEY` and use `--provider vercel-ai-gateway` ([#689](https://github.com/badlogic/pi-mono/pull/689) by [@timolins](https://github.com/timolins)) ## [0.45.3] - 2026-01-13 diff --git a/packages/coding-agent/test/model-resolver.test.ts b/packages/coding-agent/test/model-resolver.test.ts index 0471d2d4..39e48702 100644 --- a/packages/coding-agent/test/model-resolver.test.ts +++ b/packages/coding-agent/test/model-resolver.test.ts @@ -207,12 +207,12 @@ describe("default model selection", () => { }); test("findInitialModel selects ai-gateway default when available", async () => { - const aiGatewayModel: Model<"openai-completions"> = { + const aiGatewayModel: Model<"anthropic-messages"> = { id: "anthropic/claude-opus-4.5", name: "Claude Opus 4.5", - api: "openai-completions", + api: "anthropic-messages", provider: "vercel-ai-gateway", - baseUrl: "https://ai-gateway.vercel.sh/v1", + baseUrl: "https://ai-gateway.vercel.sh", reasoning: true, input: ["text", "image"], cost: { input: 5, output: 15, cacheRead: 0.5, cacheWrite: 5 },