mirror of
https://github.com/getcompanion-ai/co-mono.git
synced 2026-04-21 15:01:26 +00:00
Massive refactor of API
- Switch to function based API - Anthropic SDK style async generator - Fully typed with escape hatches for custom models
This commit is contained in:
Mario Zechner
parent
004de3c9d0
commit
66cefb236e
29 changed files with 5835 additions and 6225 deletions
|
|
@ -1,313 +1,265 @@
|
|||
import { describe, it, beforeAll, expect } from "vitest";
|
||||
import { GoogleLLM } from "../src/providers/google.js";
|
||||
import { OpenAICompletionsLLM } from "../src/providers/openai-completions.js";
|
||||
import { OpenAIResponsesLLM } from "../src/providers/openai-responses.js";
|
||||
import { AnthropicLLM } from "../src/providers/anthropic.js";
|
||||
import type { LLM, LLMOptions, Context, UserMessage, AssistantMessage } from "../src/types.js";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { complete } from "../src/generate.js";
|
||||
import { getModel } from "../src/models.js";
|
||||
import type { Api, AssistantMessage, Context, Model, OptionsForApi, UserMessage } from "../src/types.js";
|
||||
|
||||
async function testEmptyMessage<T extends LLMOptions>(llm: LLM<T>, options: T = {} as T) {
|
||||
// Test with completely empty content array
|
||||
const emptyMessage: UserMessage = {
|
||||
role: "user",
|
||||
content: []
|
||||
};
|
||||
async function testEmptyMessage<TApi extends Api>(llm: Model<TApi>, options: OptionsForApi<TApi> = {}) {
|
||||
// Test with completely empty content array
|
||||
const emptyMessage: UserMessage = {
|
||||
role: "user",
|
||||
content: [],
|
||||
};
|
||||
|
||||
const context: Context = {
|
||||
messages: [emptyMessage]
|
||||
};
|
||||
const context: Context = {
|
||||
messages: [emptyMessage],
|
||||
};
|
||||
|
||||
const response = await llm.generate(context, options);
|
||||
|
||||
// Should either handle gracefully or return an error
|
||||
expect(response).toBeDefined();
|
||||
expect(response.role).toBe("assistant");
|
||||
|
||||
// Most providers should return an error or empty response
|
||||
if (response.stopReason === "error") {
|
||||
expect(response.error).toBeDefined();
|
||||
} else {
|
||||
// If it didn't error, it should have some content or gracefully handle empty
|
||||
expect(response.content).toBeDefined();
|
||||
}
|
||||
const response = await complete(llm, context, options);
|
||||
|
||||
// Should either handle gracefully or return an error
|
||||
expect(response).toBeDefined();
|
||||
expect(response.role).toBe("assistant");
|
||||
// Should handle empty string gracefully
|
||||
if (response.stopReason === "error") {
|
||||
expect(response.error).toBeDefined();
|
||||
} else {
|
||||
expect(response.content).toBeDefined();
|
||||
}
|
||||
}
|
||||
|
||||
async function testEmptyStringMessage<T extends LLMOptions>(llm: LLM<T>, options: T = {} as T) {
|
||||
// Test with empty string content
|
||||
const context: Context = {
|
||||
messages: [{
|
||||
role: "user",
|
||||
content: ""
|
||||
}]
|
||||
};
|
||||
async function testEmptyStringMessage<TApi extends Api>(llm: Model<TApi>, options: OptionsForApi<TApi> = {}) {
|
||||
// Test with empty string content
|
||||
const context: Context = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: "",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const response = await llm.generate(context, options);
|
||||
|
||||
expect(response).toBeDefined();
|
||||
expect(response.role).toBe("assistant");
|
||||
|
||||
// Should handle empty string gracefully
|
||||
if (response.stopReason === "error") {
|
||||
expect(response.error).toBeDefined();
|
||||
} else {
|
||||
expect(response.content).toBeDefined();
|
||||
}
|
||||
const response = await complete(llm, context, options);
|
||||
|
||||
expect(response).toBeDefined();
|
||||
expect(response.role).toBe("assistant");
|
||||
|
||||
// Should handle empty string gracefully
|
||||
if (response.stopReason === "error") {
|
||||
expect(response.error).toBeDefined();
|
||||
} else {
|
||||
expect(response.content).toBeDefined();
|
||||
}
|
||||
}
|
||||
|
||||
async function testWhitespaceOnlyMessage<T extends LLMOptions>(llm: LLM<T>, options: T = {} as T) {
|
||||
// Test with whitespace-only content
|
||||
const context: Context = {
|
||||
messages: [{
|
||||
role: "user",
|
||||
content: " \n\t "
|
||||
}]
|
||||
};
|
||||
async function testWhitespaceOnlyMessage<TApi extends Api>(llm: Model<TApi>, options: OptionsForApi<TApi> = {}) {
|
||||
// Test with whitespace-only content
|
||||
const context: Context = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: " \n\t ",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const response = await llm.generate(context, options);
|
||||
|
||||
expect(response).toBeDefined();
|
||||
expect(response.role).toBe("assistant");
|
||||
|
||||
// Should handle whitespace-only gracefully
|
||||
if (response.stopReason === "error") {
|
||||
expect(response.error).toBeDefined();
|
||||
} else {
|
||||
expect(response.content).toBeDefined();
|
||||
}
|
||||
const response = await complete(llm, context, options);
|
||||
|
||||
expect(response).toBeDefined();
|
||||
expect(response.role).toBe("assistant");
|
||||
|
||||
// Should handle whitespace-only gracefully
|
||||
if (response.stopReason === "error") {
|
||||
expect(response.error).toBeDefined();
|
||||
} else {
|
||||
expect(response.content).toBeDefined();
|
||||
}
|
||||
}
|
||||
|
||||
async function testEmptyAssistantMessage<T extends LLMOptions>(llm: LLM<T>, options: T = {} as T) {
|
||||
// Test with empty assistant message in conversation flow
|
||||
// User -> Empty Assistant -> User
|
||||
const emptyAssistant: AssistantMessage = {
|
||||
role: "assistant",
|
||||
content: [],
|
||||
api: llm.getApi(),
|
||||
provider: llm.getModel().provider,
|
||||
model: llm.getModel().id,
|
||||
usage: {
|
||||
input: 10,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }
|
||||
},
|
||||
stopReason: "stop"
|
||||
};
|
||||
async function testEmptyAssistantMessage<TApi extends Api>(llm: Model<TApi>, options: OptionsForApi<TApi> = {}) {
|
||||
// Test with empty assistant message in conversation flow
|
||||
// User -> Empty Assistant -> User
|
||||
const emptyAssistant: AssistantMessage = {
|
||||
role: "assistant",
|
||||
content: [],
|
||||
api: llm.api,
|
||||
provider: llm.provider,
|
||||
model: llm.id,
|
||||
usage: {
|
||||
input: 10,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
stopReason: "stop",
|
||||
};
|
||||
|
||||
const context: Context = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: "Hello, how are you?"
|
||||
},
|
||||
emptyAssistant,
|
||||
{
|
||||
role: "user",
|
||||
content: "Please respond this time."
|
||||
}
|
||||
]
|
||||
};
|
||||
const context: Context = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: "Hello, how are you?",
|
||||
},
|
||||
emptyAssistant,
|
||||
{
|
||||
role: "user",
|
||||
content: "Please respond this time.",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const response = await llm.generate(context, options);
|
||||
|
||||
expect(response).toBeDefined();
|
||||
expect(response.role).toBe("assistant");
|
||||
|
||||
// Should handle empty assistant message in context gracefully
|
||||
if (response.stopReason === "error") {
|
||||
expect(response.error).toBeDefined();
|
||||
} else {
|
||||
expect(response.content).toBeDefined();
|
||||
expect(response.content.length).toBeGreaterThan(0);
|
||||
}
|
||||
const response = await complete(llm, context, options);
|
||||
|
||||
expect(response).toBeDefined();
|
||||
expect(response.role).toBe("assistant");
|
||||
|
||||
// Should handle empty assistant message in context gracefully
|
||||
if (response.stopReason === "error") {
|
||||
expect(response.error).toBeDefined();
|
||||
} else {
|
||||
expect(response.content).toBeDefined();
|
||||
expect(response.content.length).toBeGreaterThan(0);
|
||||
}
|
||||
}
|
||||
|
||||
describe("AI Providers Empty Message Tests", () => {
|
||||
describe.skipIf(!process.env.GEMINI_API_KEY)("Google Provider Empty Messages", () => {
|
||||
let llm: GoogleLLM;
|
||||
describe.skipIf(!process.env.GEMINI_API_KEY)("Google Provider Empty Messages", () => {
|
||||
const llm = getModel("google", "gemini-2.5-flash");
|
||||
|
||||
beforeAll(() => {
|
||||
llm = new GoogleLLM(getModel("google", "gemini-2.5-flash")!, process.env.GEMINI_API_KEY!);
|
||||
});
|
||||
it("should handle empty content array", async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty content array", async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
it("should handle empty string content", async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty string content", async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
it("should handle whitespace-only content", async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle whitespace-only content", async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
it("should handle empty assistant message in conversation", async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
});
|
||||
|
||||
it("should handle empty assistant message in conversation", async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
});
|
||||
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions Provider Empty Messages", () => {
|
||||
const llm = getModel("openai", "gpt-4o-mini");
|
||||
|
||||
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions Provider Empty Messages", () => {
|
||||
let llm: OpenAICompletionsLLM;
|
||||
it("should handle empty content array", async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
|
||||
beforeAll(() => {
|
||||
llm = new OpenAICompletionsLLM(getModel("openai", "gpt-4o-mini")!, process.env.OPENAI_API_KEY!);
|
||||
});
|
||||
it("should handle empty string content", async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty content array", async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
it("should handle whitespace-only content", async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty string content", async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
it("should handle empty assistant message in conversation", async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
});
|
||||
|
||||
it("should handle whitespace-only content", async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses Provider Empty Messages", () => {
|
||||
const llm = getModel("openai", "gpt-5-mini");
|
||||
|
||||
it("should handle empty assistant message in conversation", async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
});
|
||||
it("should handle empty content array", async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses Provider Empty Messages", () => {
|
||||
let llm: OpenAIResponsesLLM;
|
||||
it("should handle empty string content", async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
|
||||
beforeAll(() => {
|
||||
const model = getModel("openai", "gpt-5-mini");
|
||||
if (!model) {
|
||||
throw new Error("Model gpt-5-mini not found");
|
||||
}
|
||||
llm = new OpenAIResponsesLLM(model, process.env.OPENAI_API_KEY!);
|
||||
});
|
||||
it("should handle whitespace-only content", async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty content array", async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
it("should handle empty assistant message in conversation", async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
});
|
||||
|
||||
it("should handle empty string content", async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
describe.skipIf(!process.env.ANTHROPIC_OAUTH_TOKEN)("Anthropic Provider Empty Messages", () => {
|
||||
const llm = getModel("anthropic", "claude-3-5-haiku-20241022");
|
||||
|
||||
it("should handle whitespace-only content", async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
it("should handle empty content array", async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty assistant message in conversation", async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
});
|
||||
it("should handle empty string content", async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
|
||||
describe.skipIf(!process.env.ANTHROPIC_OAUTH_TOKEN)("Anthropic Provider Empty Messages", () => {
|
||||
let llm: AnthropicLLM;
|
||||
it("should handle whitespace-only content", async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
|
||||
beforeAll(() => {
|
||||
llm = new AnthropicLLM(getModel("anthropic", "claude-3-5-haiku-20241022")!, process.env.ANTHROPIC_OAUTH_TOKEN!);
|
||||
});
|
||||
it("should handle empty assistant message in conversation", async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
});
|
||||
|
||||
it("should handle empty content array", async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
describe.skipIf(!process.env.XAI_API_KEY)("xAI Provider Empty Messages", () => {
|
||||
const llm = getModel("xai", "grok-3");
|
||||
|
||||
it("should handle empty string content", async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
it("should handle empty content array", async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle whitespace-only content", async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
it("should handle empty string content", async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty assistant message in conversation", async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
});
|
||||
it("should handle whitespace-only content", async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
|
||||
// Test with xAI/Grok if available
|
||||
describe.skipIf(!process.env.XAI_API_KEY)("xAI Provider Empty Messages", () => {
|
||||
let llm: OpenAICompletionsLLM;
|
||||
it("should handle empty assistant message in conversation", async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
});
|
||||
|
||||
beforeAll(() => {
|
||||
const model = getModel("xai", "grok-3");
|
||||
if (!model) {
|
||||
throw new Error("Model grok-3 not found");
|
||||
}
|
||||
llm = new OpenAICompletionsLLM(model, process.env.XAI_API_KEY!);
|
||||
});
|
||||
describe.skipIf(!process.env.GROQ_API_KEY)("Groq Provider Empty Messages", () => {
|
||||
const llm = getModel("groq", "openai/gpt-oss-20b");
|
||||
|
||||
it("should handle empty content array", async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
it("should handle empty content array", async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty string content", async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
it("should handle empty string content", async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle whitespace-only content", async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
it("should handle whitespace-only content", async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty assistant message in conversation", async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
});
|
||||
it("should handle empty assistant message in conversation", async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
});
|
||||
|
||||
// Test with Groq if available
|
||||
describe.skipIf(!process.env.GROQ_API_KEY)("Groq Provider Empty Messages", () => {
|
||||
let llm: OpenAICompletionsLLM;
|
||||
describe.skipIf(!process.env.CEREBRAS_API_KEY)("Cerebras Provider Empty Messages", () => {
|
||||
const llm = getModel("cerebras", "gpt-oss-120b");
|
||||
|
||||
beforeAll(() => {
|
||||
const model = getModel("groq", "llama-3.3-70b-versatile");
|
||||
if (!model) {
|
||||
throw new Error("Model llama-3.3-70b-versatile not found");
|
||||
}
|
||||
llm = new OpenAICompletionsLLM(model, process.env.GROQ_API_KEY!);
|
||||
});
|
||||
it("should handle empty content array", async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty content array", async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
it("should handle empty string content", async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty string content", async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
it("should handle whitespace-only content", async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle whitespace-only content", async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty assistant message in conversation", async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
});
|
||||
|
||||
// Test with Cerebras if available
|
||||
describe.skipIf(!process.env.CEREBRAS_API_KEY)("Cerebras Provider Empty Messages", () => {
|
||||
let llm: OpenAICompletionsLLM;
|
||||
|
||||
beforeAll(() => {
|
||||
const model = getModel("cerebras", "gpt-oss-120b");
|
||||
if (!model) {
|
||||
throw new Error("Model gpt-oss-120b not found");
|
||||
}
|
||||
llm = new OpenAICompletionsLLM(model, process.env.CEREBRAS_API_KEY!);
|
||||
});
|
||||
|
||||
it("should handle empty content array", async () => {
|
||||
await testEmptyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty string content", async () => {
|
||||
await testEmptyStringMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle whitespace-only content", async () => {
|
||||
await testWhitespaceOnlyMessage(llm);
|
||||
});
|
||||
|
||||
it("should handle empty assistant message in conversation", async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
});
|
||||
});
|
||||
it("should handle empty assistant message in conversation", async () => {
|
||||
await testEmptyAssistantMessage(llm);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue