mirror of
https://github.com/getcompanion-ai/co-mono.git
synced 2026-04-20 01:00:24 +00:00
Fix PR #689: Add changelog attribution, coding-agent changelog, fix test types, add provider to test suites
- Fix ai/CHANGELOG.md: add PR link and author attribution - Add coding-agent/CHANGELOG.md entry for vercel-ai-gateway provider - Fix model-resolver.test.ts: use anthropic-messages API type to match generated models - Add vercel-ai-gateway to test suites: tokens, abort, empty, context-overflow, unicode-surrogate, tool-call-without-result, image-tool-result, total-tokens, image-limits
This commit is contained in:
parent
bde59e6c0d
commit
19f3c23f6d
12 changed files with 162 additions and 4 deletions
|
|
@ -4,7 +4,7 @@
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
- Added Vercel AI Gateway provider with model discovery and `AI_GATEWAY_API_KEY` env support.
|
- Added Vercel AI Gateway provider with model discovery and `AI_GATEWAY_API_KEY` env support ([#689](https://github.com/badlogic/pi-mono/pull/689) by [@timolins](https://github.com/timolins))
|
||||||
|
|
||||||
## [0.45.3] - 2026-01-13
|
## [0.45.3] - 2026-01-13
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -172,6 +172,18 @@ describe("AI Providers Abort Tests", () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider Abort", () => {
|
||||||
|
const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash");
|
||||||
|
|
||||||
|
it("should abort mid-stream", { retry: 3 }, async () => {
|
||||||
|
await testAbortSignal(llm);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should handle immediate abort", { retry: 3 }, async () => {
|
||||||
|
await testImmediateAbort(llm);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
// Google Gemini CLI / Antigravity share the same provider, so one test covers both
|
// Google Gemini CLI / Antigravity share the same provider, so one test covers both
|
||||||
describe("Google Gemini CLI Provider Abort", () => {
|
describe("Google Gemini CLI Provider Abort", () => {
|
||||||
it.skipIf(!geminiCliToken)("should abort mid-stream", { retry: 3 }, async () => {
|
it.skipIf(!geminiCliToken)("should abort mid-stream", { retry: 3 }, async () => {
|
||||||
|
|
|
||||||
|
|
@ -412,6 +412,21 @@ describe("Context overflow error handling", () => {
|
||||||
}, 120000);
|
}, 120000);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// =============================================================================
|
||||||
|
// Vercel AI Gateway - Unified API for multiple providers
|
||||||
|
// =============================================================================
|
||||||
|
|
||||||
|
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway", () => {
|
||||||
|
it("google/gemini-2.5-flash via AI Gateway - should detect overflow via isContextOverflow", async () => {
|
||||||
|
const model = getModel("vercel-ai-gateway", "google/gemini-2.5-flash");
|
||||||
|
const result = await testContextOverflow(model, process.env.AI_GATEWAY_API_KEY!);
|
||||||
|
logResult(result);
|
||||||
|
|
||||||
|
expect(result.stopReason).toBe("error");
|
||||||
|
expect(isContextOverflow(result.response, model.contextWindow)).toBe(true);
|
||||||
|
}, 120000);
|
||||||
|
});
|
||||||
|
|
||||||
// =============================================================================
|
// =============================================================================
|
||||||
// OpenRouter - Multiple backend providers
|
// OpenRouter - Multiple backend providers
|
||||||
// Expected pattern: "maximum context length is X tokens"
|
// Expected pattern: "maximum context length is X tokens"
|
||||||
|
|
|
||||||
|
|
@ -342,6 +342,26 @@ describe("AI Providers Empty Message Tests", () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider Empty Messages", () => {
|
||||||
|
const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash");
|
||||||
|
|
||||||
|
it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {
|
||||||
|
await testEmptyMessage(llm);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should handle empty string content", { retry: 3, timeout: 30000 }, async () => {
|
||||||
|
await testEmptyStringMessage(llm);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should handle whitespace-only content", { retry: 3, timeout: 30000 }, async () => {
|
||||||
|
await testWhitespaceOnlyMessage(llm);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should handle empty assistant message in conversation", { retry: 3, timeout: 30000 }, async () => {
|
||||||
|
await testEmptyAssistantMessage(llm);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider Empty Messages", () => {
|
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider Empty Messages", () => {
|
||||||
const llm = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0");
|
const llm = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0");
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -841,6 +841,49 @@ describe("Image Limits E2E Tests", () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
// Vercel AI Gateway (google/gemini-2.5-flash)
|
||||||
|
// -------------------------------------------------------------------------
|
||||||
|
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway (google/gemini-2.5-flash)", () => {
|
||||||
|
const model = getModel("vercel-ai-gateway", "google/gemini-2.5-flash");
|
||||||
|
|
||||||
|
it("should accept a small number of images (5)", async () => {
|
||||||
|
const result = await testImageCount(model, 5, smallImage);
|
||||||
|
expect(result.success, result.error).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should find maximum image count limit", { timeout: 600000 }, async () => {
|
||||||
|
const { limit, lastError } = await findLimit((count) => testImageCount(model, count, smallImage), 10, 100, 10);
|
||||||
|
console.log(`\n Vercel AI Gateway max images: ~${limit} (last error: ${lastError})`);
|
||||||
|
expect(limit).toBeGreaterThanOrEqual(5);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should find maximum image size limit", { timeout: 600000 }, async () => {
|
||||||
|
const MB = 1024 * 1024;
|
||||||
|
const sizes = [5, 10, 15, 20];
|
||||||
|
|
||||||
|
let lastSuccess = 0;
|
||||||
|
let lastError: string | undefined;
|
||||||
|
|
||||||
|
for (const sizeMB of sizes) {
|
||||||
|
console.log(` Testing size: ${sizeMB}MB...`);
|
||||||
|
const imageBase64 = generateImageWithSize(sizeMB * MB, `size-${sizeMB}mb.png`);
|
||||||
|
const result = await testImageSize(model, imageBase64);
|
||||||
|
if (result.success) {
|
||||||
|
lastSuccess = sizeMB;
|
||||||
|
console.log(` SUCCESS`);
|
||||||
|
} else {
|
||||||
|
lastError = result.error;
|
||||||
|
console.log(` FAILED: ${result.error?.substring(0, 100)}`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`\n Vercel AI Gateway max image size: ~${lastSuccess}MB (last error: ${lastError})`);
|
||||||
|
expect(lastSuccess).toBeGreaterThanOrEqual(5);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
// -------------------------------------------------------------------------
|
// -------------------------------------------------------------------------
|
||||||
// Amazon Bedrock (claude-sonnet-4-5)
|
// Amazon Bedrock (claude-sonnet-4-5)
|
||||||
// Limits: 100 images (Anthropic), 5MB per image, 8000px max dimension
|
// Limits: 100 images (Anthropic), 5MB per image, 8000px max dimension
|
||||||
|
|
|
||||||
|
|
@ -274,6 +274,18 @@ describe("Tool Results with Images", () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider (google/gemini-2.5-flash)", () => {
|
||||||
|
const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash");
|
||||||
|
|
||||||
|
it("should handle tool result with only image", { retry: 3, timeout: 30000 }, async () => {
|
||||||
|
await handleToolWithImageResult(llm);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should handle tool result with text and image", { retry: 3, timeout: 30000 }, async () => {
|
||||||
|
await handleToolWithTextAndImageResult(llm);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider (claude-sonnet-4-5)", () => {
|
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider (claude-sonnet-4-5)", () => {
|
||||||
const llm = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0");
|
const llm = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0");
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -159,6 +159,14 @@ describe("Token Statistics on Abort", () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider", () => {
|
||||||
|
const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash");
|
||||||
|
|
||||||
|
it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {
|
||||||
|
await testTokensOnAbort(llm);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
// =========================================================================
|
// =========================================================================
|
||||||
// OAuth-based providers (credentials from ~/.pi/agent/oauth.json)
|
// OAuth-based providers (credentials from ~/.pi/agent/oauth.json)
|
||||||
// =========================================================================
|
// =========================================================================
|
||||||
|
|
|
||||||
|
|
@ -179,6 +179,14 @@ describe("Tool Call Without Result Tests", () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider", () => {
|
||||||
|
const model = getModel("vercel-ai-gateway", "google/gemini-2.5-flash");
|
||||||
|
|
||||||
|
it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {
|
||||||
|
await testToolCallWithoutResult(model);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider", () => {
|
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider", () => {
|
||||||
const model = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0");
|
const model = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0");
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -348,6 +348,29 @@ describe("totalTokens field", () => {
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// =========================================================================
|
||||||
|
// Vercel AI Gateway
|
||||||
|
// =========================================================================
|
||||||
|
|
||||||
|
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway", () => {
|
||||||
|
it(
|
||||||
|
"google/gemini-2.5-flash - should return totalTokens equal to sum of components",
|
||||||
|
{ retry: 3, timeout: 60000 },
|
||||||
|
async () => {
|
||||||
|
const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash");
|
||||||
|
|
||||||
|
console.log(`\nVercel AI Gateway / ${llm.id}:`);
|
||||||
|
const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.AI_GATEWAY_API_KEY });
|
||||||
|
|
||||||
|
logUsage("First request", first);
|
||||||
|
logUsage("Second request", second);
|
||||||
|
|
||||||
|
assertTotalTokensEqualsComponents(first);
|
||||||
|
assertTotalTokensEqualsComponents(second);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
// =========================================================================
|
// =========================================================================
|
||||||
// OpenRouter - Multiple backend providers
|
// OpenRouter - Multiple backend providers
|
||||||
// =========================================================================
|
// =========================================================================
|
||||||
|
|
|
||||||
|
|
@ -634,6 +634,22 @@ describe("AI Providers Unicode Surrogate Pair Tests", () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("Vercel AI Gateway Provider Unicode Handling", () => {
|
||||||
|
const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash");
|
||||||
|
|
||||||
|
it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {
|
||||||
|
await testEmojiInToolResults(llm);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should handle real-world LinkedIn comment data with emoji", { retry: 3, timeout: 30000 }, async () => {
|
||||||
|
await testRealWorldLinkedInData(llm);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should handle unpaired high surrogate (0xD83D) in tool results", { retry: 3, timeout: 30000 }, async () => {
|
||||||
|
await testUnpairedHighSurrogate(llm);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider Unicode Handling", () => {
|
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider Unicode Handling", () => {
|
||||||
const llm = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0");
|
const llm = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0");
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
- Extension example: `summarize.ts` for summarizing conversations using custom UI and an external model
|
- Extension example: `summarize.ts` for summarizing conversations using custom UI and an external model
|
||||||
|
- Vercel AI Gateway provider support: set `AI_GATEWAY_API_KEY` and use `--provider vercel-ai-gateway` ([#689](https://github.com/badlogic/pi-mono/pull/689) by [@timolins](https://github.com/timolins))
|
||||||
|
|
||||||
## [0.45.3] - 2026-01-13
|
## [0.45.3] - 2026-01-13
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -207,12 +207,12 @@ describe("default model selection", () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
test("findInitialModel selects ai-gateway default when available", async () => {
|
test("findInitialModel selects ai-gateway default when available", async () => {
|
||||||
const aiGatewayModel: Model<"openai-completions"> = {
|
const aiGatewayModel: Model<"anthropic-messages"> = {
|
||||||
id: "anthropic/claude-opus-4.5",
|
id: "anthropic/claude-opus-4.5",
|
||||||
name: "Claude Opus 4.5",
|
name: "Claude Opus 4.5",
|
||||||
api: "openai-completions",
|
api: "anthropic-messages",
|
||||||
provider: "vercel-ai-gateway",
|
provider: "vercel-ai-gateway",
|
||||||
baseUrl: "https://ai-gateway.vercel.sh/v1",
|
baseUrl: "https://ai-gateway.vercel.sh",
|
||||||
reasoning: true,
|
reasoning: true,
|
||||||
input: ["text", "image"],
|
input: ["text", "image"],
|
||||||
cost: { input: 5, output: 15, cacheRead: 0.5, cacheWrite: 5 },
|
cost: { input: 5, output: 15, cacheRead: 0.5, cacheWrite: 5 },
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue