diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 02894a48..c5785f6b 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -5,6 +5,7 @@ ### Added - Added `GOOGLE_APPLICATION_CREDENTIALS` env var support for Vertex AI credential detection (standard for CI/production). +- Added `supportsUsageInStreaming` compatibility flag for OpenAI-compatible providers that reject `stream_options: { include_usage: true }`. Defaults to `true`. Set to `false` in model config for providers like gatewayz.ai. ([#596](https://github.com/badlogic/pi-mono/pull/596) by [@XesGaDeus](https://github.com/XesGaDeus)) ### Fixed diff --git a/packages/ai/src/providers/openai-completions.ts b/packages/ai/src/providers/openai-completions.ts index e37f46a1..091996ff 100644 --- a/packages/ai/src/providers/openai-completions.ts +++ b/packages/ai/src/providers/openai-completions.ts @@ -370,9 +370,12 @@ function buildParams(model: Model<"openai-completions">, context: Context, optio model: model.id, messages, stream: true, - stream_options: { include_usage: true }, }; + if (compat.supportsUsageInStreaming !== false) { + (params as any).stream_options = { include_usage: true }; + } + if (compat.supportsStore) { params.store = false; } @@ -657,6 +660,7 @@ function detectCompatFromUrl(baseUrl: string): Required { supportsStore: !isNonStandard, supportsDeveloperRole: !isNonStandard, supportsReasoningEffort: !isGrok, + supportsUsageInStreaming: true, maxTokensField: useMaxTokens ? "max_tokens" : "max_completion_tokens", requiresToolResultName: isMistral, requiresAssistantAfterToolResult: false, // Mistral no longer requires this as of Dec 2024 @@ -677,6 +681,7 @@ function getCompat(model: Model<"openai-completions">): Required { supportsStore: model.compat.supportsStore ?? detected.supportsStore, supportsDeveloperRole: model.compat.supportsDeveloperRole ?? detected.supportsDeveloperRole, supportsReasoningEffort: model.compat.supportsReasoningEffort ?? detected.supportsReasoningEffort, + supportsUsageInStreaming: model.compat.supportsUsageInStreaming ?? detected.supportsUsageInStreaming, maxTokensField: model.compat.maxTokensField ?? detected.maxTokensField, requiresToolResultName: model.compat.requiresToolResultName ?? detected.requiresToolResultName, requiresAssistantAfterToolResult: diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 75ba8a6b..e8061016 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -207,6 +207,8 @@ export interface OpenAICompat { supportsDeveloperRole?: boolean; /** Whether the provider supports `reasoning_effort`. Default: auto-detected from URL. */ supportsReasoningEffort?: boolean; + /** Whether the provider supports `stream_options: { include_usage: true }` for token usage in streaming responses. Default: true. */ + supportsUsageInStreaming?: boolean; /** Which field to use for max tokens. Default: auto-detected from URL. */ maxTokensField?: "max_completion_tokens" | "max_tokens"; /** Whether tool results require the `name` field. Default: auto-detected from URL. */ diff --git a/packages/ai/test/.temp-images/dim-1000.png b/packages/ai/test/.temp-images/dim-1000.png new file mode 100644 index 00000000..c503fad1 Binary files /dev/null and b/packages/ai/test/.temp-images/dim-1000.png differ diff --git a/packages/ai/test/.temp-images/dim-10000.png b/packages/ai/test/.temp-images/dim-10000.png new file mode 100644 index 00000000..dd964a1f Binary files /dev/null and b/packages/ai/test/.temp-images/dim-10000.png differ diff --git a/packages/ai/test/.temp-images/dim-2000.png b/packages/ai/test/.temp-images/dim-2000.png new file mode 100644 index 00000000..f13b0c7f Binary files /dev/null and b/packages/ai/test/.temp-images/dim-2000.png differ diff --git a/packages/ai/test/.temp-images/dim-4000.png b/packages/ai/test/.temp-images/dim-4000.png new file mode 100644 index 00000000..bf425df2 Binary files /dev/null and b/packages/ai/test/.temp-images/dim-4000.png differ diff --git a/packages/ai/test/.temp-images/dim-6000.png b/packages/ai/test/.temp-images/dim-6000.png new file mode 100644 index 00000000..dc308967 Binary files /dev/null and b/packages/ai/test/.temp-images/dim-6000.png differ diff --git a/packages/ai/test/.temp-images/dim-8000.png b/packages/ai/test/.temp-images/dim-8000.png new file mode 100644 index 00000000..f6f0a6cf Binary files /dev/null and b/packages/ai/test/.temp-images/dim-8000.png differ diff --git a/packages/ai/test/.temp-images/size-1mb.png b/packages/ai/test/.temp-images/size-1mb.png index b6cda334..18778cf8 100644 Binary files a/packages/ai/test/.temp-images/size-1mb.png and b/packages/ai/test/.temp-images/size-1mb.png differ diff --git a/packages/ai/test/.temp-images/size-2mb.png b/packages/ai/test/.temp-images/size-2mb.png index 8051b400..9b620612 100644 Binary files a/packages/ai/test/.temp-images/size-2mb.png and b/packages/ai/test/.temp-images/size-2mb.png differ diff --git a/packages/ai/test/.temp-images/size-3mb.png b/packages/ai/test/.temp-images/size-3mb.png new file mode 100644 index 00000000..47359b6b Binary files /dev/null and b/packages/ai/test/.temp-images/size-3mb.png differ diff --git a/packages/ai/test/.temp-images/size-4mb.png b/packages/ai/test/.temp-images/size-4mb.png new file mode 100644 index 00000000..7583ab95 Binary files /dev/null and b/packages/ai/test/.temp-images/size-4mb.png differ diff --git a/packages/ai/test/.temp-images/small.png b/packages/ai/test/.temp-images/small.png index 4443fcc9..f1059066 100644 Binary files a/packages/ai/test/.temp-images/small.png and b/packages/ai/test/.temp-images/small.png differ