test(ai): Add empty message tests for all providers

- Test handling of empty content arrays
- Test handling of empty string content
- Test handling of whitespace-only content
- All providers handle these edge cases gracefully
This commit is contained in:
Mario Zechner 2025-09-02 02:03:06 +02:00
parent 0fbb0921bb
commit 0ac05a0676
9 changed files with 256 additions and 18 deletions

12
package-lock.json generated
View file

@ -2716,10 +2716,10 @@
},
"packages/agent": {
"name": "@mariozechner/pi-agent",
"version": "0.5.26",
"version": "0.5.27",
"license": "MIT",
"dependencies": {
"@mariozechner/pi-tui": "^0.5.19",
"@mariozechner/pi-tui": "^0.5.26",
"@types/glob": "^8.1.0",
"chalk": "^5.5.0",
"glob": "^11.0.3",
@ -3098,7 +3098,7 @@
},
"packages/ai": {
"name": "@mariozechner/pi-ai",
"version": "0.5.26",
"version": "0.5.27",
"license": "MIT",
"dependencies": {
"@anthropic-ai/sdk": "^0.60.0",
@ -3134,10 +3134,10 @@
},
"packages/pods": {
"name": "@mariozechner/pi",
"version": "0.5.26",
"version": "0.5.27",
"license": "MIT",
"dependencies": {
"@mariozechner/pi-agent": "^0.5.19",
"@mariozechner/pi-agent": "^0.5.26",
"chalk": "^5.5.0"
},
"bin": {
@ -3150,7 +3150,7 @@
},
"packages/tui": {
"name": "@mariozechner/pi-tui",
"version": "0.5.26",
"version": "0.5.27",
"license": "MIT",
"dependencies": {
"@types/mime-types": "^2.1.4",

View file

@ -1,12 +1,12 @@
{
"name": "@mariozechner/pi-agent",
"version": "0.5.26",
"version": "0.5.27",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@mariozechner/pi-agent",
"version": "0.5.26",
"version": "0.5.27",
"license": "MIT",
"dependencies": {
"@mariozechner/tui": "^0.1.1",

View file

@ -1,6 +1,6 @@
{
"name": "@mariozechner/pi-agent",
"version": "0.5.26",
"version": "0.5.27",
"description": "General-purpose agent with tool calling and session persistence",
"type": "module",
"bin": {
@ -18,7 +18,7 @@
"prepublishOnly": "npm run clean && npm run build"
},
"dependencies": {
"@mariozechner/pi-tui": "^0.5.26",
"@mariozechner/pi-tui": "^0.5.27",
"@types/glob": "^8.1.0",
"chalk": "^5.5.0",
"glob": "^11.0.3",

View file

@ -1,6 +1,6 @@
{
"name": "@mariozechner/pi-ai",
"version": "0.5.26",
"version": "0.5.27",
"description": "Unified LLM API with automatic model discovery and provider configuration",
"type": "module",
"main": "./dist/index.js",

View file

@ -0,0 +1,238 @@
import { describe, it, beforeAll, expect } from "vitest";
import { GoogleLLM } from "../src/providers/google.js";
import { OpenAICompletionsLLM } from "../src/providers/openai-completions.js";
import { OpenAIResponsesLLM } from "../src/providers/openai-responses.js";
import { AnthropicLLM } from "../src/providers/anthropic.js";
import type { LLM, LLMOptions, Context, UserMessage } from "../src/types.js";
import { getModel } from "../src/models.js";
async function testEmptyMessage<T extends LLMOptions>(llm: LLM<T>, options: T = {} as T) {
// Test with completely empty content array
const emptyMessage: UserMessage = {
role: "user",
content: []
};
const context: Context = {
messages: [emptyMessage]
};
const response = await llm.generate(context, options);
// Should either handle gracefully or return an error
expect(response).toBeDefined();
expect(response.role).toBe("assistant");
// Most providers should return an error or empty response
if (response.stopReason === "error") {
expect(response.error).toBeDefined();
} else {
// If it didn't error, it should have some content or gracefully handle empty
expect(response.content).toBeDefined();
}
}
async function testEmptyStringMessage<T extends LLMOptions>(llm: LLM<T>, options: T = {} as T) {
// Test with empty string content
const context: Context = {
messages: [{
role: "user",
content: ""
}]
};
const response = await llm.generate(context, options);
expect(response).toBeDefined();
expect(response.role).toBe("assistant");
// Should handle empty string gracefully
if (response.stopReason === "error") {
expect(response.error).toBeDefined();
} else {
expect(response.content).toBeDefined();
}
}
async function testWhitespaceOnlyMessage<T extends LLMOptions>(llm: LLM<T>, options: T = {} as T) {
// Test with whitespace-only content
const context: Context = {
messages: [{
role: "user",
content: " \n\t "
}]
};
const response = await llm.generate(context, options);
expect(response).toBeDefined();
expect(response.role).toBe("assistant");
// Should handle whitespace-only gracefully
if (response.stopReason === "error") {
expect(response.error).toBeDefined();
} else {
expect(response.content).toBeDefined();
}
}
describe("AI Providers Empty Message Tests", () => {
describe.skipIf(!process.env.GEMINI_API_KEY)("Google Provider Empty Messages", () => {
let llm: GoogleLLM;
beforeAll(() => {
llm = new GoogleLLM(getModel("google", "gemini-2.5-flash")!, process.env.GEMINI_API_KEY!);
});
it("should handle empty content array", async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", async () => {
await testWhitespaceOnlyMessage(llm);
});
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions Provider Empty Messages", () => {
let llm: OpenAICompletionsLLM;
beforeAll(() => {
llm = new OpenAICompletionsLLM(getModel("openai", "gpt-4o-mini")!, process.env.OPENAI_API_KEY!);
});
it("should handle empty content array", async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", async () => {
await testWhitespaceOnlyMessage(llm);
});
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses Provider Empty Messages", () => {
let llm: OpenAIResponsesLLM;
beforeAll(() => {
const model = getModel("openai", "gpt-5-mini");
if (!model) {
throw new Error("Model gpt-5-mini not found");
}
llm = new OpenAIResponsesLLM(model, process.env.OPENAI_API_KEY!);
});
it("should handle empty content array", async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", async () => {
await testWhitespaceOnlyMessage(llm);
});
});
describe.skipIf(!process.env.ANTHROPIC_OAUTH_TOKEN)("Anthropic Provider Empty Messages", () => {
let llm: AnthropicLLM;
beforeAll(() => {
llm = new AnthropicLLM(getModel("anthropic", "claude-3-5-haiku-20241022")!, process.env.ANTHROPIC_OAUTH_TOKEN!);
});
it("should handle empty content array", async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", async () => {
await testWhitespaceOnlyMessage(llm);
});
});
// Test with xAI/Grok if available
describe.skipIf(!process.env.XAI_API_KEY)("xAI Provider Empty Messages", () => {
let llm: OpenAICompletionsLLM;
beforeAll(() => {
const model = getModel("xai", "grok-3");
if (!model) {
throw new Error("Model grok-3 not found");
}
llm = new OpenAICompletionsLLM(model, process.env.XAI_API_KEY!);
});
it("should handle empty content array", async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", async () => {
await testWhitespaceOnlyMessage(llm);
});
});
// Test with Groq if available
describe.skipIf(!process.env.GROQ_API_KEY)("Groq Provider Empty Messages", () => {
let llm: OpenAICompletionsLLM;
beforeAll(() => {
const model = getModel("groq", "llama-3.3-70b-versatile");
if (!model) {
throw new Error("Model llama-3.3-70b-versatile not found");
}
llm = new OpenAICompletionsLLM(model, process.env.GROQ_API_KEY!);
});
it("should handle empty content array", async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", async () => {
await testWhitespaceOnlyMessage(llm);
});
});
// Test with Cerebras if available
describe.skipIf(!process.env.CEREBRAS_API_KEY)("Cerebras Provider Empty Messages", () => {
let llm: OpenAICompletionsLLM;
beforeAll(() => {
const model = getModel("cerebras", "gpt-oss-120b");
if (!model) {
throw new Error("Model gpt-oss-120b not found");
}
llm = new OpenAICompletionsLLM(model, process.env.CEREBRAS_API_KEY!);
});
it("should handle empty content array", async () => {
await testEmptyMessage(llm);
});
it("should handle empty string content", async () => {
await testEmptyStringMessage(llm);
});
it("should handle whitespace-only content", async () => {
await testWhitespaceOnlyMessage(llm);
});
});
});

View file

@ -1,12 +1,12 @@
{
"name": "@mariozechner/pi",
"version": "0.5.26",
"version": "0.5.27",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@mariozechner/pi",
"version": "0.5.26",
"version": "0.5.27",
"license": "MIT",
"dependencies": {
"@ai-sdk/openai": "^2.0.5",

View file

@ -1,6 +1,6 @@
{
"name": "@mariozechner/pi",
"version": "0.5.26",
"version": "0.5.27",
"description": "CLI tool for managing vLLM deployments on GPU pods",
"type": "module",
"bin": {
@ -34,7 +34,7 @@
"node": ">=20.0.0"
},
"dependencies": {
"@mariozechner/pi-agent": "^0.5.26",
"@mariozechner/pi-agent": "^0.5.27",
"chalk": "^5.5.0"
},
"devDependencies": {}

View file

@ -1,12 +1,12 @@
{
"name": "@mariozechner/tui",
"version": "0.5.26",
"version": "0.5.27",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@mariozechner/tui",
"version": "0.5.26",
"version": "0.5.27",
"license": "MIT",
"dependencies": {
"@types/mime-types": "^2.1.4",

View file

@ -1,6 +1,6 @@
{
"name": "@mariozechner/pi-tui",
"version": "0.5.26",
"version": "0.5.27",
"description": "Terminal User Interface library with differential rendering for efficient text-based applications",
"type": "module",
"main": "dist/index.js",