getcompanion-ai/co-mono
1
0
Fork 0
mirror of https://github.com/getcompanion-ai/co-mono.git synced 2026-04-15 03:00:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(ai): Migrate tests to Vitest and add provider test coverage

Browse source 
- Switch from Node.js test runner to Vitest for better DX
- Add test suites for Grok, Groq, Cerebras, and OpenRouter providers
- Add Ollama test suite with automatic server lifecycle management
- Include thinking mode and multi-turn tests for all providers
- Remove example files (consolidated into test suite)
- Add VS Code test configuration
This commit is contained in:
Mario Zechner 2025-08-29 21:32:45 +02:00
parent da66a97ea7
commit 3f36051bc6
14 changed files with 1319 additions and 636 deletions
Download patch file Download diff file Expand all files Collapse all files

1
CLAUDE.md
View file

@ -1,5 +1,6 @@
- When receiving the first user message, you MUST read the following files in full, in parallel:
- README.md
- packages/ai/README.md
- packages/tui/README.md
- packages/agent/README.md
- packages/pods/README.md

1084
package-lock.json generated
View file

File diff suppressed because it is too large Load diff

8
packages/ai/package.json
View file

@ -15,7 +15,9 @@
"generate-models": "npx tsx scripts/generate-models.ts",
"build": "npm run generate-models && tsc -p tsconfig.build.json && cp src/models.json dist/models.json",
"check": "biome check --write .",
"test": "npx tsx --test test/providers.test.ts",
"test": "vitest",
"test:ui": "vitest --ui",
"test:old": "npx tsx --test test/providers.test.ts",
"extract-models": "npx tsx scripts/extract-openai-models.ts",
"prepublishOnly": "npm run clean && npm run models && npm run build"
},
@ -45,6 +47,8 @@
"node": ">=20.0.0"
},
"devDependencies": {
"@types/node": "^24.3.0"
"@types/node": "^24.3.0",
"@vitest/ui": "^3.2.4",
"vitest": "^3.2.4"
}
}

50
packages/ai/src/providers/openai-completions.ts
View file

@ -45,8 +45,8 @@ export class OpenAICompletionsLLM implements LLM<OpenAICompletionsLLMOptions> {
stream_options: { include_usage: true },
};
// Cerebras doesn't like the "store" field
if (!this.client.baseURL?.includes("cerebras.ai")) {
// Cerebras/xAI dont like the "store" field
if (!this.client.baseURL?.includes("cerebras.ai") || this.client.baseURL?.includes("api.x.ai")) {
(params as any).store = false;
}
@ -66,7 +66,7 @@ export class OpenAICompletionsLLM implements LLM<OpenAICompletionsLLMOptions> {
params.tool_choice = options.toolChoice;
}
if (options?.reasoningEffort && this.isReasoningModel()) {
if (options?.reasoningEffort && this.isReasoningModel() && !this.model.toLowerCase().includes("grok")) {
params.reasoning_effort = options.reasoningEffort;
}
@ -77,14 +77,7 @@ export class OpenAICompletionsLLM implements LLM<OpenAICompletionsLLMOptions> {
let content = "";
let reasoningContent = "";
let reasoningField: "reasoning" | "reasoning_content" | null = null;
const toolCallsMap = new Map<
number,
{
id: string;
name: string;
arguments: string;
}
>();
const parsedToolCalls: { id: string; name: string; arguments: string }[] = [];
let usage: TokenUsage = {
input: 0,
output: 0,
@ -97,7 +90,9 @@ export class OpenAICompletionsLLM implements LLM<OpenAICompletionsLLMOptions> {
if (chunk.usage) {
usage = {
input: chunk.usage.prompt_tokens || 0,
output: chunk.usage.completion_tokens || 0,
output:
(chunk.usage.completion_tokens || 0) +
(chunk.usage.completion_tokens_details?.reasoning_tokens || 0),
cacheRead: chunk.usage.prompt_tokens_details?.cached_tokens || 0,
cacheWrite: 0,
};
@ -122,7 +117,7 @@ export class OpenAICompletionsLLM implements LLM<OpenAICompletionsLLMOptions> {
blockType = "text";
}
// Handle LLAMA.cpp reasoning_content
// Handle reasoning_content field
if (
(choice.delta as any).reasoning_content !== null &&
(choice.delta as any).reasoning_content !== undefined
@ -137,7 +132,7 @@ export class OpenAICompletionsLLM implements LLM<OpenAICompletionsLLMOptions> {
blockType = "thinking";
}
// Handle Ollama reasoning field
// Handle reasoning field
if ((choice.delta as any).reasoning !== null && (choice.delta as any).reasoning !== undefined) {
if (blockType === "text") {
options?.onText?.("", true);
@ -160,21 +155,22 @@ export class OpenAICompletionsLLM implements LLM<OpenAICompletionsLLMOptions> {
blockType = null;
}
for (const toolCall of choice.delta.tool_calls) {
const index = toolCall.index;
if (!toolCallsMap.has(index)) {
toolCallsMap.set(index, {
if (
parsedToolCalls.length === 0 ||
(toolCall.id !== undefined && parsedToolCalls[parsedToolCalls.length - 1].id !== toolCall.id)
) {
parsedToolCalls.push({
id: toolCall.id || "",
name: toolCall.function?.name || "",
arguments: "",
});
}
const existing = toolCallsMap.get(index)!;
if (toolCall.id) existing.id = toolCall.id;
if (toolCall.function?.name) existing.name = toolCall.function.name;
const current = parsedToolCalls[parsedToolCalls.length - 1];
if (toolCall.id) current.id = toolCall.id;
if (toolCall.function?.name) current.name = toolCall.function.name;
if (toolCall.function?.arguments) {
existing.arguments += toolCall.function.arguments;
current.arguments += toolCall.function.arguments;
}
}
}
@ -195,7 +191,7 @@ export class OpenAICompletionsLLM implements LLM<OpenAICompletionsLLMOptions> {
}
// Convert tool calls map to array
const toolCalls: ToolCall[] = Array.from(toolCallsMap.values()).map((tc) => ({
const toolCalls: ToolCall[] = parsedToolCalls.map((tc) => ({
id: tc.id,
name: tc.name,
arguments: JSON.parse(tc.arguments),
@ -232,8 +228,12 @@ export class OpenAICompletionsLLM implements LLM<OpenAICompletionsLLMOptions> {
// Add system prompt if provided
if (systemPrompt) {
// Cerebras doesn't like the "developer" role
const role = this.isReasoningModel() && !this.client.baseURL?.includes("cerebras.ai") ? "developer" : "system";
// Cerebras/xAi don't like the "developer" role
const useDeveloperRole =
this.isReasoningModel() &&
!this.client.baseURL?.includes("cerebras.ai") &&
!this.client.baseURL?.includes("api.x.ai");
const role = useDeveloperRole ? "developer" : "system";
params.push({ role: role, content: systemPrompt });
}

67
packages/ai/test/examples/anthropic.ts
View file

@ -1,67 +0,0 @@
import chalk from "chalk";
import { readFileSync } from "fs";
import { fileURLToPath } from "url";
import { dirname, join } from "path";
import { AnthropicLLM, AnthropicLLMOptions } from "../../src/providers/anthropic";
import { Context, Tool } from "../../src/types";
// Define a simple calculator tool
const tools: Tool[] = [
{
name: "calculate",
description: "Perform a mathematical calculation",
parameters: {
type: "object" as const,
properties: {
expression: {
type: "string",
description: "The mathematical expression to evaluate"
}
},
required: ["expression"]
}
}
];
const options: AnthropicLLMOptions = {
onText: (t, complete) => process.stdout.write(t + (complete ? "\n" : "")),
onThinking: (t, complete) => process.stdout.write(chalk.dim(t + (complete ? "\n" : ""))),
thinking: { enabled: true }
};
const ai = new AnthropicLLM("claude-sonnet-4-0", process.env.ANTHROPIC_OAUTH_TOKEN ?? process.env.ANTHROPIC_API_KEY);
const context: Context = {
systemPrompt: "You are a helpful assistant that can use tools to answer questions.",
messages: [
{
role: "user",
content: "Think about birds briefly. Then give me a list of 10 birds. Finally, calculate 42 * 17 + 123 and 453 + 434 in parallel using the calculator tool.",
}
],
tools
}
let msg = await ai.complete(context, options)
context.messages.push(msg);
console.log();
console.log(chalk.yellow(JSON.stringify(msg, null, 2)));
for (const toolCall of msg.toolCalls || []) {
if (toolCall.name === "calculate") {
const expression = toolCall.arguments.expression;
const result = eval(expression);
context.messages.push({
role: "toolResult",
content: `The result of ${expression} is ${result}.`,
toolCallId: toolCall.id,
isError: false
});
}
}
msg = await ai.complete(context, options);
console.log();
console.log(chalk.yellow(JSON.stringify(msg, null, 2)));

65
packages/ai/test/examples/cerebras-completions.ts
View file

@ -1,65 +0,0 @@
import chalk from "chalk";
import { Context, Tool } from "../../src/types";
import { OpenAICompletionsLLM, OpenAICompletionsLLMOptions } from "../../src/providers/openai-completions";
// Define a simple calculator tool
const tools: Tool[] = [
{
name: "calculate",
description: "Perform a mathematical calculation",
parameters: {
type: "object" as const,
properties: {
expression: {
type: "string",
description: "The mathematical expression to evaluate"
}
},
required: ["expression"]
}
}
];
const options: OpenAICompletionsLLMOptions = {
onText: (t, complete) => process.stdout.write(t + (complete ? "\n" : "")),
onThinking: (t, complete) => process.stdout.write(chalk.dim(t + (complete ? "\n" : ""))),
reasoningEffort: "medium",
toolChoice: "auto"
};
const ai = new OpenAICompletionsLLM("gpt-oss-120b", process.env.CEREBRAS_API_KEY, "https://api.cerebras.ai/v1");
const context: Context = {
systemPrompt: "You are a helpful assistant that can use tools to answer questions.",
messages: [
{
role: "user",
content: "Think about birds briefly. Then give me a list of 10 birds. Finally, calculate 42 * 17 + 123 and 453 + 434 in parallel using the calculator tool. You must use the tool to answer both math questions.",
}
],
tools
}
while (true) {
let msg = await ai.complete(context, options)
context.messages.push(msg);
console.log();
for (const toolCall of msg.toolCalls || []) {
if (toolCall.name === "calculate") {
const expression = toolCall.arguments.expression;
const result = eval(expression);
context.messages.push({
role: "toolResult",
content: `The result of ${expression} is ${result}.`,
toolCallId: toolCall.id,
isError: false
});
}
}
if (msg.stopReason != "toolUse") break;
}
console.log();
console.log(chalk.yellow(JSON.stringify(context.messages, null, 2)));

65
packages/ai/test/examples/gemini.ts
View file

@ -1,65 +0,0 @@
import chalk from "chalk";
import { GeminiLLM, GeminiLLMOptions } from "../../src/providers/gemini.js";
import { Context, Tool } from "../../src/types.js";
// Define a simple calculator tool
const tools: Tool[] = [
{
name: "calculate",
description: "Perform a mathematical calculation",
parameters: {
type: "object" as const,
properties: {
expression: {
type: "string",
description: "The mathematical expression to evaluate"
}
},
required: ["expression"]
}
}
];
const options: GeminiLLMOptions = {
onText: (t, complete) => process.stdout.write(t + (complete ? "\n" : "")),
onThinking: (t, complete) => process.stdout.write(chalk.dim(t + (complete ? "\n" : ""))),
toolChoice: "auto",
thinking: {
enabled: true,
budgetTokens: -1 // Dynamic thinking
}
};
const ai = new GeminiLLM("gemini-2.5-flash", process.env.GEMINI_API_KEY);
const context: Context = {
systemPrompt: "You are a helpful assistant that can use tools to answer questions.",
messages: [
{
role: "user",
content: "Think about birds briefly. Then give me a list of 10 birds. Finally, calculate 42 * 17 + 123 and 453 + 434 in parallel using the calculator tool.",
}
],
tools
}
let msg = await ai.complete(context, options)
context.messages.push(msg);
console.log();
console.log(chalk.yellow(JSON.stringify(msg, null, 2)));
for (const toolCall of msg.toolCalls || []) {
if (toolCall.name === "calculate") {
const expression = toolCall.arguments.expression;
const result = eval(expression);
context.messages.push({
role: "toolResult",
content: `The result of ${expression} is ${result}.`,
toolCallId: toolCall.id,
isError: false
});
}
}
msg = await ai.complete(context, options);
console.log();
console.log(chalk.yellow(JSON.stringify(msg, null, 2)));

66
packages/ai/test/examples/groq-completions.ts
View file

@ -1,66 +0,0 @@
import chalk from "chalk";
import { Context, Tool } from "../../src/types";
import { OpenAICompletionsLLM, OpenAICompletionsLLMOptions } from "../../src/providers/openai-completions";
// Define a simple calculator tool
const tools: Tool[] = [
{
name: "calculate",
description: "Perform a mathematical calculation",
parameters: {
type: "object" as const,
properties: {
expression: {
type: "string",
description: "The mathematical expression to evaluate"
}
},
required: ["expression"]
}
}
];
const options: OpenAICompletionsLLMOptions = {
onText: (t, complete) => process.stdout.write(t + (complete ? "\n" : "")),
onThinking: (t, complete) => process.stdout.write(chalk.dim(t + (complete ? "\n" : ""))),
reasoningEffort: "medium",
toolChoice: "auto"
};
const ai = new OpenAICompletionsLLM("openai/gpt-oss-20b", process.env.GROQ_API_KEY, "https://api.groq.com/openai/v1");
const context: Context = {
systemPrompt: "You are a helpful assistant that can use tools to answer questions.",
messages: [
{
role: "user",
content: "Think about birds briefly. Then give me a list of 10 birds. Finally, calculate 42 * 17 + 123 and 453 + 434 in parallel using the calculator tool.",
}
],
tools
}
while (true) {
let msg = await ai.complete(context, options)
context.messages.push(msg);
console.log();
console.log(chalk.yellow(JSON.stringify(msg, null, 2)));
for (const toolCall of msg.toolCalls || []) {
if (toolCall.name === "calculate") {
const expression = toolCall.arguments.expression;
const result = eval(expression);
context.messages.push({
role: "toolResult",
content: `The result of ${expression} is ${result}.`,
toolCallId: toolCall.id,
isError: false
});
}
}
if (msg.stopReason != "toolUse") break;
}
console.log();
console.log(chalk.yellow(JSON.stringify(context.messages, null, 2)));

66
packages/ai/test/examples/ollama-completions.ts
View file

@ -1,66 +0,0 @@
import chalk from "chalk";
import { Context, Tool } from "../../src/types";
import { OpenAICompletionsLLM, OpenAICompletionsLLMOptions } from "../../src/providers/openai-completions";
// Define a simple calculator tool
const tools: Tool[] = [
{
name: "calculate",
description: "Perform a mathematical calculation",
parameters: {
type: "object" as const,
properties: {
expression: {
type: "string",
description: "The mathematical expression to evaluate"
}
},
required: ["expression"]
}
}
];
const options: OpenAICompletionsLLMOptions = {
onText: (t, complete) => process.stdout.write(t + (complete ? "\n" : "")),
onThinking: (t, complete) => process.stdout.write(chalk.dim(t + (complete ? "\n" : ""))),
reasoningEffort: "medium",
toolChoice: "auto"
};
const ai = new OpenAICompletionsLLM("gpt-oss:20b", "dummy", "http://localhost:11434/v1");
const context: Context = {
systemPrompt: "You are a helpful assistant that can use tools to answer questions.",
messages: [
{
role: "user",
content: "Think about birds briefly. Then give me a list of 10 birds. Finally, calculate 42 * 17 + 123 and 453 + 434 in parallel using the calculator tool.",
}
],
tools
}
while (true) {
let msg = await ai.complete(context, options)
context.messages.push(msg);
console.log();
console.log(chalk.yellow(JSON.stringify(msg, null, 2)));
for (const toolCall of msg.toolCalls || []) {
if (toolCall.name === "calculate") {
const expression = toolCall.arguments.expression;
const result = eval(expression);
context.messages.push({
role: "toolResult",
content: `The result of ${expression} is ${result}.`,
toolCallId: toolCall.id,
isError: false
});
}
}
if (msg.stopReason == "stop") break;
}
console.log();
console.log(chalk.yellow(JSON.stringify(context.messages, null, 2)));

65
packages/ai/test/examples/openai-completions.ts
View file

@ -1,65 +0,0 @@
import chalk from "chalk";
import { Context, Tool } from "../../src/types";
import { OpenAICompletionsLLM, OpenAICompletionsLLMOptions } from "../../src/providers/openai-completions";
// Define a simple calculator tool
const tools: Tool[] = [
{
name: "calculate",
description: "Perform a mathematical calculation",
parameters: {
type: "object" as const,
properties: {
expression: {
type: "string",
description: "The mathematical expression to evaluate"
}
},
required: ["expression"]
}
}
];
const options: OpenAICompletionsLLMOptions = {
onText: (t, complete) => process.stdout.write(t + (complete ? "\n" : "")),
onThinking: (t, complete) => process.stdout.write(chalk.dim(t + (complete ? "\n" : ""))),
reasoningEffort: "medium",
toolChoice: "auto"
};
const ai = new OpenAICompletionsLLM("gpt-5-mini");
const context: Context = {
systemPrompt: "You are a helpful assistant that can use tools to answer questions.",
messages: [
{
role: "user",
content: "Think about birds briefly. Then give me a list of 10 birds. Finally, calculate 42 * 17 + 123 and 453 + 434 in parallel using the calculator tool.",
}
],
tools
}
let msg = await ai.complete(context, options)
context.messages.push(msg);
console.log();
console.log(chalk.yellow(JSON.stringify(msg, null, 2)));
for (const toolCall of msg.toolCalls || []) {
if (toolCall.name === "calculate") {
const expression = toolCall.arguments.expression;
const result = eval(expression);
context.messages.push({
role: "toolResult",
content: `The result of ${expression} is ${result}.`,
toolCallId: toolCall.id,
isError: false
});
}
}
msg = await ai.complete(context, options);
console.log();
console.log(chalk.yellow(JSON.stringify(msg, null, 2)));

60
packages/ai/test/examples/openai-responses.ts
View file

@ -1,60 +0,0 @@
import chalk from "chalk";
import { OpenAIResponsesLLMOptions, OpenAIResponsesLLM } from "../../src/providers/openai-responses.js";
import type { Context, Tool } from "../../src/types.js";
// Define a simple calculator tool
const tools: Tool[] = [
{
name: "calculate",
description: "Perform a mathematical calculation",
parameters: {
type: "object" as const,
properties: {
expression: {
type: "string",
description: "The mathematical expression to evaluate"
}
},
required: ["expression"]
}
}
];
const ai = new OpenAIResponsesLLM("gpt-5");
const context: Context = {
messages: [
{
role: "user",
content: "Think about birds briefly. Then give me a list of 10 birds. Finally, calculate 42 * 17 + 123 and 453 + 434 in parallel using the calculator tool.",
}
],
tools,
}
const options: OpenAIResponsesLLMOptions = {
onText: (t, complete) => process.stdout.write(t + (complete ? "\n" : "")),
onThinking: (t, complete) => process.stdout.write(chalk.dim(t + (complete ? "\n" : ""))),
reasoningEffort: "low",
reasoningSummary: "auto"
};
let msg = await ai.complete(context, options)
context.messages.push(msg);
console.log();
console.log(chalk.yellow(JSON.stringify(msg, null, 2)));
for (const toolCall of msg.toolCalls || []) {
if (toolCall.name === "calculate") {
const expression = toolCall.arguments.expression;
const result = eval(expression);
context.messages.push({
role: "toolResult",
content: `The result of ${expression} is ${result}.`,
toolCallId: toolCall.id,
isError: false
});
}
}
msg = await ai.complete(context, options);
console.log();
console.log(chalk.yellow(JSON.stringify(msg, null, 2)));

65
packages/ai/test/examples/openrouter-completions.ts
View file

@ -1,65 +0,0 @@
import chalk from "chalk";
import { Context, Tool } from "../../src/types";
import { OpenAICompletionsLLM, OpenAICompletionsLLMOptions } from "../../src/providers/openai-completions";
// Define a simple calculator tool
const tools: Tool[] = [
{
name: "calculate",
description: "Perform a mathematical calculation",
parameters: {
type: "object" as const,
properties: {
expression: {
type: "string",
description: "The mathematical expression to evaluate"
}
},
required: ["expression"]
}
}
];
const options: OpenAICompletionsLLMOptions = {
onText: (t, complete) => process.stdout.write(t + (complete ? "\n" : "")),
onThinking: (t, complete) => process.stdout.write(chalk.dim(t + (complete ? "\n" : ""))),
reasoningEffort: "medium",
toolChoice: "auto"
};
const ai = new OpenAICompletionsLLM("z-ai/glm-4.5", process.env.OPENROUTER_API_KEY, "https://openrouter.ai/api/v1");
const context: Context = {
systemPrompt: "You are a helpful assistant that can use tools to answer questions.",
messages: [
{
role: "user",
content: "Think about birds briefly. Then give me a list of 10 birds. Finally, calculate 42 * 17 + 123 and 453 + 434 in parallel using the calculator tool.",
}
],
tools
}
while (true) {
let msg = await ai.complete(context, options)
context.messages.push(msg);
console.log();
for (const toolCall of msg.toolCalls || []) {
if (toolCall.name === "calculate") {
const expression = toolCall.arguments.expression;
const result = eval(expression);
context.messages.push({
role: "toolResult",
content: `The result of ${expression} is ${result}.`,
toolCallId: toolCall.id,
isError: false
});
}
}
if (msg.stopReason != "toolUse") break;
}
console.log();
console.log(chalk.yellow(JSON.stringify(context.messages, null, 2)));

284
packages/ai/test/providers.test.ts
View file

@ -1,11 +1,10 @@
#!/usr/bin/env node --test
import { describe, it, before } from "node:test";
import assert from "node:assert";
import { describe, it, beforeAll, afterAll, expect } from "vitest";
import { GeminiLLM } from "../src/providers/gemini.js";
import { OpenAICompletionsLLM } from "../src/providers/openai-completions.js";
import { OpenAIResponsesLLM } from "../src/providers/openai-responses.js";
import { AnthropicLLM } from "../src/providers/anthropic.js";
import type { LLM, LLMOptions, Context, Tool, AssistantMessage } from "../src/types.js";
import { spawn, ChildProcess, execSync } from "child_process";
// Calculator tool definition (same as examples)
const calculatorTool: Tool = {
@ -36,12 +35,12 @@ async function basicTextGeneration<T extends LLMOptions>(llm: LLM<T>) {
const response = await llm.complete(context);
assert.strictEqual(response.role, "assistant");
assert.ok(response.content);
assert.ok(response.usage.input > 0);
assert.ok(response.usage.output > 0);
assert.ok(!response.error);
assert.ok(response.content.includes("Hello test successful"), `Response content should match exactly. Got: ${response.content}`);
expect(response.role).toBe("assistant");
expect(response.content).toBeTruthy();
expect(response.usage.input).toBeGreaterThan(0);
expect(response.usage.output).toBeGreaterThan(0);
expect(response.error).toBeFalsy();
expect(response.content).toContain("Hello test successful");
}
async function handleToolCall<T extends LLMOptions>(llm: LLM<T>) {
@ -55,11 +54,12 @@ async function handleToolCall<T extends LLMOptions>(llm: LLM<T>) {
};
const response = await llm.complete(context);
assert.ok(response.stopReason == "toolUse", "Response should indicate tool use");
assert.ok(response.toolCalls && response.toolCalls.length > 0, "Response should include tool calls");
const toolCall = response.toolCalls[0];
assert.strictEqual(toolCall.name, "calculator");
assert.ok(toolCall.id);
expect(response.stopReason).toBe("toolUse");
expect(response.toolCalls).toBeTruthy();
expect(response.toolCalls!.length).toBeGreaterThan(0);
const toolCall = response.toolCalls![0];
expect(toolCall.name).toBe("calculator");
expect(toolCall.id).toBeTruthy();
}
async function handleStreaming<T extends LLMOptions>(llm: LLM<T>) {
@ -77,9 +77,9 @@ async function handleStreaming<T extends LLMOptions>(llm: LLM<T>) {
}
} as T);
assert.ok(textChunks.length > 0);
assert.ok(textCompleted);
assert.ok(response.content);
expect(textChunks.length).toBeGreaterThan(0);
expect(textCompleted).toBe(true);
expect(response.content).toBeTruthy();
}
async function handleThinking<T extends LLMOptions>(llm: LLM<T>, options: T, requireThinking: boolean = true) {
@ -96,14 +96,11 @@ async function handleThinking<T extends LLMOptions>(llm: LLM<T>, options: T, req
...options
});
assert.ok(response.content, "Response should have content");
expect(response.content).toBeTruthy();
// For providers that should always return thinking when enabled
if (requireThinking) {
assert.ok(
thinkingChunks.length > 0 || response.thinking,
`LLM MUST return thinking content when thinking is enabled. Got ${thinkingChunks.length} streaming chars, thinking field: ${response.thinking?.length || 0} chars`
);
expect(thinkingChunks.length > 0 || !!response.thinking).toBe(true);
}
}
@ -123,17 +120,15 @@ async function multiTurn<T extends LLMOptions>(llm: LLM<T>, thinkingOptions: T)
const firstResponse = await llm.complete(context, thinkingOptions);
// Verify we got either thinking content or tool calls (or both)
const hasThinking = firstResponse.thinking;
const hasThinking = firstResponse.thinking !== undefined && firstResponse.thinking.length > 0;
const hasToolCalls = firstResponse.toolCalls && firstResponse.toolCalls.length > 0;
assert.ok(
hasThinking || hasToolCalls,
`First turn MUST include either thinking or tool calls. Got thinking: ${hasThinking}, tool calls: ${hasToolCalls}`
);
expect(hasThinking || hasToolCalls).toBe(true);
// If we got tool calls, verify they're correct
if (hasToolCalls) {
assert.ok(firstResponse.toolCalls && firstResponse.toolCalls.length > 0, "First turn should include tool calls");
expect(firstResponse.toolCalls).toBeTruthy();
expect(firstResponse.toolCalls!.length).toBeGreaterThan(0);
}
// If we have thinking with tool calls, we should have thinkingSignature for proper multi-turn context
@ -142,7 +137,7 @@ async function multiTurn<T extends LLMOptions>(llm: LLM<T>, thinkingOptions: T)
// For now, we'll just check if it exists when both are present
// Some providers may not support thinkingSignature yet
if (firstResponse.thinkingSignature !== undefined) {
assert.ok(firstResponse.thinkingSignature, "Response with thinking and tools should include thinkingSignature");
expect(firstResponse.thinkingSignature).toBeTruthy();
}
}
@ -151,9 +146,9 @@ async function multiTurn<T extends LLMOptions>(llm: LLM<T>, thinkingOptions: T)
// Process tool calls and add results
for (const toolCall of firstResponse.toolCalls || []) {
assert.strictEqual(toolCall.name, "calculator", "Tool call should be for calculator");
assert.ok(toolCall.id, "Tool call must have an ID");
assert.ok(toolCall.arguments, "Tool call must have arguments");
expect(toolCall.name).toBe("calculator");
expect(toolCall.id).toBeTruthy();
expect(toolCall.arguments).toBeTruthy();
const { a, b, operation } = toolCall.arguments;
let result: number;
@ -206,22 +201,21 @@ async function multiTurn<T extends LLMOptions>(llm: LLM<T>, thinkingOptions: T)
}
}
assert.ok(finalResponse, "Should get a final response with content");
assert.ok(finalResponse.content, "Final response should have content");
assert.strictEqual(finalResponse.role, "assistant");
expect(finalResponse).toBeTruthy();
expect(finalResponse!.content).toBeTruthy();
expect(finalResponse!.role).toBe("assistant");
// The final response should reference the calculations
assert.ok(
finalResponse.content.includes("714") || finalResponse.content.includes("887"),
`Final response should include calculation results. Got: ${finalResponse.content}`
);
expect(
finalResponse!.content!.includes("714") || finalResponse!.content!.includes("887")
).toBe(true);
}
describe("AI Providers E2E Tests", () => {
describe("Gemini Provider", { skip: !process.env.GEMINI_API_KEY }, () => {
describe.skipIf(!process.env.GEMINI_API_KEY)("Gemini Provider", () => {
let llm: GeminiLLM;
before(() => {
beforeAll(() => {
llm = new GeminiLLM("gemini-2.5-flash", process.env.GEMINI_API_KEY!);
});
@ -246,10 +240,10 @@ describe("AI Providers E2E Tests", () => {
});
});
describe("OpenAI Completions Provider", { skip: !process.env.OPENAI_API_KEY }, () => {
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions Provider", () => {
let llm: OpenAICompletionsLLM;
before(() => {
beforeAll(() => {
llm = new OpenAICompletionsLLM("gpt-4o-mini", process.env.OPENAI_API_KEY!);
});
@ -266,10 +260,10 @@ describe("AI Providers E2E Tests", () => {
});
});
describe("OpenAI Responses Provider", { skip: !process.env.OPENAI_API_KEY }, () => {
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses Provider", () => {
let llm: OpenAIResponsesLLM;
before(() => {
beforeAll(() => {
llm = new OpenAIResponsesLLM("gpt-5-mini", process.env.OPENAI_API_KEY!);
});
@ -286,8 +280,6 @@ describe("AI Providers E2E Tests", () => {
});
it("should handle thinking mode", async () => {
// OpenAI Responses API may not always return thinking even when requested
// This is model-dependent behavior
await handleThinking(llm, {reasoningEffort: "medium"}, false);
});
@ -296,10 +288,10 @@ describe("AI Providers E2E Tests", () => {
});
});
describe("Anthropic Provider", { skip: !process.env.ANTHROPIC_OAUTH_TOKEN }, () => {
describe.skipIf(!process.env.ANTHROPIC_OAUTH_TOKEN)("Anthropic Provider", () => {
let llm: AnthropicLLM;
before(() => {
beforeAll(() => {
llm = new AnthropicLLM("claude-sonnet-4-0", process.env.ANTHROPIC_OAUTH_TOKEN!);
});
@ -323,4 +315,198 @@ describe("AI Providers E2E Tests", () => {
await multiTurn(llm, {thinking: { enabled: true, budgetTokens: 2048 }});
});
});
describe.skipIf(!process.env.GROK_API_KEY)("Grok Provider (via OpenAI Completions)", () => {
let llm: OpenAICompletionsLLM;
beforeAll(() => {
llm = new OpenAICompletionsLLM("grok-code-fast-1", process.env.GROK_API_KEY!, "https://api.x.ai/v1");
});
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, {reasoningEffort: "medium"}, false);
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, {reasoningEffort: "medium"});
});
});
describe.skipIf(!process.env.GROQ_API_KEY)("Groq Provider (via OpenAI Completions)", () => {
let llm: OpenAICompletionsLLM;
beforeAll(() => {
llm = new OpenAICompletionsLLM("openai/gpt-oss-20b", process.env.GROQ_API_KEY!, "https://api.groq.com/openai/v1");
});
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, {reasoningEffort: "medium"}, false);
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, {reasoningEffort: "medium"});
});
});
describe.skipIf(!process.env.CEREBRAS_API_KEY)("Cerebras Provider (via OpenAI Completions)", () => {
let llm: OpenAICompletionsLLM;
beforeAll(() => {
llm = new OpenAICompletionsLLM("gpt-oss-120b", process.env.CEREBRAS_API_KEY!, "https://api.cerebras.ai/v1");
});
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, {reasoningEffort: "medium"}, false);
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, {reasoningEffort: "medium"});
});
});
describe.skipIf(!process.env.OPENROUTER_API_KEY)("OpenRouter Provider (via OpenAI Completions)", () => {
let llm: OpenAICompletionsLLM;
beforeAll(() => {
llm = new OpenAICompletionsLLM("z-ai/glm-4.5", process.env.OPENROUTER_API_KEY!, "https://openrouter.ai/api/v1");
});
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, {reasoningEffort: "medium"}, false);
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, {reasoningEffort: "medium"});
});
});
// Check if ollama is installed
let ollamaInstalled = false;
try {
execSync("which ollama", { stdio: "ignore" });
ollamaInstalled = true;
} catch {
ollamaInstalled = false;
}
describe.skipIf(!ollamaInstalled)("Ollama Provider (via OpenAI Completions)", () => {
let llm: OpenAICompletionsLLM;
let ollamaProcess: ChildProcess | null = null;
beforeAll(async () => {
// Check if model is available, if not pull it
try {
execSync("ollama list | grep -q 'gpt-oss:20b'", { stdio: "ignore" });
} catch {
console.log("Pulling gpt-oss:20b model for Ollama tests...");
try {
execSync("ollama pull gpt-oss:20b", { stdio: "inherit" });
} catch (e) {
console.warn("Failed to pull gpt-oss:20b model, tests will be skipped");
return;
}
}
// Start ollama server
ollamaProcess = spawn("ollama", ["serve"], {
detached: false,
stdio: "ignore"
});
// Wait for server to be ready
await new Promise<void>((resolve) => {
const checkServer = async () => {
try {
const response = await fetch("http://localhost:11434/api/tags");
if (response.ok) {
resolve();
} else {
setTimeout(checkServer, 500);
}
} catch {
setTimeout(checkServer, 500);
}
};
setTimeout(checkServer, 1000); // Initial delay
});
llm = new OpenAICompletionsLLM("gpt-oss:20b", "dummy", "http://localhost:11434/v1");
}, 30000); // 30 second timeout for setup
afterAll(() => {
// Kill ollama server
if (ollamaProcess) {
ollamaProcess.kill("SIGTERM");
ollamaProcess = null;
}
});
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, {reasoningEffort: "medium"}, false);
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, {reasoningEffort: "medium"});
});
});
});

9
packages/ai/vitest.config.ts Normal file
View file

@ -0,0 +1,9 @@
import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
globals: true,
environment: 'node',
testTimeout: 30000, // 30 seconds for API calls
}
});