mirror of
https://github.com/getcompanion-ai/co-mono.git
synced 2026-04-19 20:00:41 +00:00
feat(ai): Add start event emission to all providers
- Emit start event with model and provider info after creating stream - Add abort signal tests for all providers - Update README abort signal section to reflect non-throwing API - Fix model references in README examples
This commit is contained in:
parent
8d4edf6458
commit
a132b8140c
6 changed files with 167 additions and 20 deletions
|
|
@ -160,24 +160,25 @@ const response = await llm.complete({
|
||||||
```typescript
|
```typescript
|
||||||
const controller = new AbortController();
|
const controller = new AbortController();
|
||||||
|
|
||||||
// Abort after 5 seconds
|
// Abort after 2 seconds
|
||||||
setTimeout(() => controller.abort(), 5000);
|
setTimeout(() => controller.abort(), 2000);
|
||||||
|
|
||||||
try {
|
const response = await llm.complete({
|
||||||
const response = await llm.complete({
|
|
||||||
messages: [{ role: 'user', content: 'Write a long story' }]
|
messages: [{ role: 'user', content: 'Write a long story' }]
|
||||||
}, {
|
}, {
|
||||||
signal: controller.signal,
|
signal: controller.signal,
|
||||||
onEvent: (event) => {
|
onEvent: (event) => {
|
||||||
if (event.type === 'text_delta') {
|
if (event.type === 'text_delta') {
|
||||||
process.stdout.write(event.delta);
|
process.stdout.write(event.delta);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
} catch (error) {
|
|
||||||
if (error.name === 'AbortError') {
|
// Check if the request was aborted
|
||||||
console.log('Request was aborted');
|
if (response.stopReason === 'error' && response.error) {
|
||||||
}
|
console.log('Request was aborted:', response.error);
|
||||||
|
} else {
|
||||||
|
console.log('Request completed successfully');
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -206,7 +207,7 @@ await llm.complete(context, {
|
||||||
|
|
||||||
### Google Gemini Thinking
|
### Google Gemini Thinking
|
||||||
```typescript
|
```typescript
|
||||||
const llm = createLLM('google', 'gemini-2.0-flash-thinking-exp');
|
const llm = createLLM('google', 'gemini-2.5-pro');
|
||||||
|
|
||||||
await llm.complete(context, {
|
await llm.complete(context, {
|
||||||
thinking: { enabled: true }
|
thinking: { enabled: true }
|
||||||
|
|
@ -220,14 +221,14 @@ await llm.complete(context, {
|
||||||
import { OpenAICompletionsLLM } from '@mariozechner/pi-ai';
|
import { OpenAICompletionsLLM } from '@mariozechner/pi-ai';
|
||||||
|
|
||||||
const model = {
|
const model = {
|
||||||
id: 'llama3.1:8b',
|
id: 'gpt-oss:20b',
|
||||||
provider: 'ollama',
|
provider: 'ollama',
|
||||||
baseUrl: 'http://localhost:11434/v1',
|
baseUrl: 'http://localhost:11434/v1',
|
||||||
reasoning: false,
|
reasoning: false,
|
||||||
input: ['text'],
|
input: ['text'],
|
||||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||||
contextWindow: 8192,
|
contextWindow: 126000,
|
||||||
maxTokens: 4096,
|
maxTokens: 32000,
|
||||||
name: 'Llama 3.1 8B'
|
name: 'Llama 3.1 8B'
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -129,6 +129,8 @@ export class AnthropicLLM implements LLM<AnthropicLLMOptions> {
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
options?.onEvent?.({ type: "start", model: this.modelInfo.id, provider: this.modelInfo.provider });
|
||||||
|
|
||||||
let blockType: "text" | "thinking" | "toolUse" | "other" = "other";
|
let blockType: "text" | "thinking" | "toolUse" | "other" = "other";
|
||||||
let blockContent = "";
|
let blockContent = "";
|
||||||
let toolCall: (ToolCall & { partialJson: string }) | null = null;
|
let toolCall: (ToolCall & { partialJson: string }) | null = null;
|
||||||
|
|
|
||||||
|
|
@ -89,6 +89,14 @@ export class GoogleLLM implements LLM<GoogleLLMOptions> {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Abort signal
|
||||||
|
if (options?.signal) {
|
||||||
|
if (options.signal.aborted) {
|
||||||
|
throw new Error("Request aborted");
|
||||||
|
}
|
||||||
|
config.abortSignal = options.signal;
|
||||||
|
}
|
||||||
|
|
||||||
// Build the request parameters
|
// Build the request parameters
|
||||||
const params: GenerateContentParameters = {
|
const params: GenerateContentParameters = {
|
||||||
model: this.model.id,
|
model: this.model.id,
|
||||||
|
|
@ -98,6 +106,8 @@ export class GoogleLLM implements LLM<GoogleLLMOptions> {
|
||||||
|
|
||||||
const stream = await this.client.models.generateContentStream(params);
|
const stream = await this.client.models.generateContentStream(params);
|
||||||
|
|
||||||
|
options?.onEvent?.({ type: "start", model: this.model.id, provider: this.model.provider });
|
||||||
|
|
||||||
const blocks: AssistantMessage["content"] = [];
|
const blocks: AssistantMessage["content"] = [];
|
||||||
let currentBlock: TextContent | ThinkingContent | null = null;
|
let currentBlock: TextContent | ThinkingContent | null = null;
|
||||||
let usage: Usage = {
|
let usage: Usage = {
|
||||||
|
|
|
||||||
|
|
@ -92,6 +92,8 @@ export class OpenAICompletionsLLM implements LLM<OpenAICompletionsLLMOptions> {
|
||||||
signal: options?.signal,
|
signal: options?.signal,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
options?.onEvent?.({ type: "start", model: this.modelInfo.id, provider: this.modelInfo.provider });
|
||||||
|
|
||||||
const blocks: AssistantMessage["content"] = [];
|
const blocks: AssistantMessage["content"] = [];
|
||||||
let currentBlock: TextContent | ThinkingContent | (ToolCall & { partialArgs?: string }) | null = null;
|
let currentBlock: TextContent | ThinkingContent | (ToolCall & { partialArgs?: string }) | null = null;
|
||||||
let usage: Usage = {
|
let usage: Usage = {
|
||||||
|
|
|
||||||
|
|
@ -85,6 +85,8 @@ export class OpenAIResponsesLLM implements LLM<OpenAIResponsesLLMOptions> {
|
||||||
signal: options?.signal,
|
signal: options?.signal,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
options?.onEvent?.({ type: "start", model: this.modelInfo.id, provider: this.modelInfo.provider });
|
||||||
|
|
||||||
const outputItems: (ResponseReasoningItem | ResponseOutputMessage | ResponseFunctionToolCall)[] = []; // any for function_call items
|
const outputItems: (ResponseReasoningItem | ResponseOutputMessage | ResponseFunctionToolCall)[] = []; // any for function_call items
|
||||||
let currentTextAccum = ""; // For delta accumulation
|
let currentTextAccum = ""; // For delta accumulation
|
||||||
let currentThinkingAccum = ""; // For delta accumulation
|
let currentThinkingAccum = ""; // For delta accumulation
|
||||||
|
|
@ -184,9 +186,25 @@ export class OpenAIResponsesLLM implements LLM<OpenAIResponsesLLMOptions> {
|
||||||
} satisfies AssistantMessage;
|
} satisfies AssistantMessage;
|
||||||
options?.onEvent?.({ type: "error", error: errorOutput.error || "Unknown error" });
|
options?.onEvent?.({ type: "error", error: errorOutput.error || "Unknown error" });
|
||||||
return errorOutput;
|
return errorOutput;
|
||||||
|
} else if (event.type === "response.failed") {
|
||||||
|
const errorOutput = {
|
||||||
|
role: "assistant",
|
||||||
|
content: [],
|
||||||
|
provider: this.modelInfo.provider,
|
||||||
|
model: this.modelInfo.id,
|
||||||
|
usage,
|
||||||
|
stopReason: "error",
|
||||||
|
error: "Unknown error",
|
||||||
|
} satisfies AssistantMessage;
|
||||||
|
options?.onEvent?.({ type: "error", error: errorOutput.error || "Unknown error" });
|
||||||
|
return errorOutput;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (options?.signal?.aborted) {
|
||||||
|
throw new Error("Request was aborted");
|
||||||
|
}
|
||||||
|
|
||||||
// Convert output items to blocks
|
// Convert output items to blocks
|
||||||
const blocks: AssistantMessage["content"] = [];
|
const blocks: AssistantMessage["content"] = [];
|
||||||
|
|
||||||
|
|
|
||||||
114
packages/ai/test/abort.test.ts
Normal file
114
packages/ai/test/abort.test.ts
Normal file
|
|
@ -0,0 +1,114 @@
|
||||||
|
import { describe, it, beforeAll, expect } from "vitest";
|
||||||
|
import { GoogleLLM } from "../src/providers/google.js";
|
||||||
|
import { OpenAICompletionsLLM } from "../src/providers/openai-completions.js";
|
||||||
|
import { OpenAIResponsesLLM } from "../src/providers/openai-responses.js";
|
||||||
|
import { AnthropicLLM } from "../src/providers/anthropic.js";
|
||||||
|
import type { LLM, LLMOptions, Context } from "../src/types.js";
|
||||||
|
import { getModel } from "../src/models.js";
|
||||||
|
|
||||||
|
async function testAbortSignal<T extends LLMOptions>(llm: LLM<T>) {
|
||||||
|
const controller = new AbortController();
|
||||||
|
|
||||||
|
// Abort after 100ms
|
||||||
|
setTimeout(() => controller.abort(), 1000);
|
||||||
|
|
||||||
|
const context: Context = {
|
||||||
|
messages: [{
|
||||||
|
role: "user",
|
||||||
|
content: "Write a very long story about a dragon that lives in a mountain. Include lots of details about the dragon's appearance, its daily life, the treasures it guards, and its interactions with nearby villages. Make it at least 1000 words long."
|
||||||
|
}]
|
||||||
|
};
|
||||||
|
|
||||||
|
const response = await llm.complete(context, {
|
||||||
|
signal: controller.signal
|
||||||
|
} as T);
|
||||||
|
|
||||||
|
// If we get here without throwing, the abort didn't work
|
||||||
|
expect(response.stopReason).toBe("error");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function testImmediateAbort<T extends LLMOptions>(llm: LLM<T>) {
|
||||||
|
const controller = new AbortController();
|
||||||
|
|
||||||
|
// Abort immediately
|
||||||
|
controller.abort();
|
||||||
|
|
||||||
|
const context: Context = {
|
||||||
|
messages: [{ role: "user", content: "Hello" }]
|
||||||
|
};
|
||||||
|
|
||||||
|
const response = await llm.complete(context, {
|
||||||
|
signal: controller.signal
|
||||||
|
} as T);
|
||||||
|
expect(response.stopReason).toBe("error");
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("AI Providers Abort Tests", () => {
|
||||||
|
describe.skipIf(!process.env.GEMINI_API_KEY)("Google Provider Abort", () => {
|
||||||
|
let llm: GoogleLLM;
|
||||||
|
|
||||||
|
beforeAll(() => {
|
||||||
|
llm = new GoogleLLM(getModel("google", "gemini-2.5-flash")!, process.env.GEMINI_API_KEY!);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should abort mid-stream", async () => {
|
||||||
|
await testAbortSignal(llm);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should handle immediate abort", async () => {
|
||||||
|
await testImmediateAbort(llm);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions Provider Abort", () => {
|
||||||
|
let llm: OpenAICompletionsLLM;
|
||||||
|
|
||||||
|
beforeAll(() => {
|
||||||
|
llm = new OpenAICompletionsLLM(getModel("openai", "gpt-4o-mini")!, process.env.OPENAI_API_KEY!);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should abort mid-stream", async () => {
|
||||||
|
await testAbortSignal(llm);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should handle immediate abort", async () => {
|
||||||
|
await testImmediateAbort(llm);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses Provider Abort", () => {
|
||||||
|
let llm: OpenAIResponsesLLM;
|
||||||
|
|
||||||
|
beforeAll(() => {
|
||||||
|
const model = getModel("openai", "gpt-5-mini");
|
||||||
|
if (!model) {
|
||||||
|
throw new Error("Model not found");
|
||||||
|
}
|
||||||
|
llm = new OpenAIResponsesLLM(model, process.env.OPENAI_API_KEY!);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should abort mid-stream", async () => {
|
||||||
|
await testAbortSignal(llm);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should handle immediate abort", async () => {
|
||||||
|
await testImmediateAbort(llm);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic Provider Abort", () => {
|
||||||
|
let llm: AnthropicLLM;
|
||||||
|
|
||||||
|
beforeAll(() => {
|
||||||
|
llm = new AnthropicLLM(getModel("anthropic", "claude-3-5-haiku-latest")!, process.env.ANTHROPIC_API_KEY!);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should abort mid-stream", async () => {
|
||||||
|
await testAbortSignal(llm);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should handle immediate abort", async () => {
|
||||||
|
await testImmediateAbort(llm);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
Loading…
Add table
Add a link
Reference in a new issue