mirror of
https://github.com/getcompanion-ai/co-mono.git
synced 2026-04-19 20:00:41 +00:00
feat(ai): Add Amazon Bedrock provider (#494)
Adds support for Amazon Bedrock with Claude models including: - Full streaming support via Converse API - Reasoning/thinking support for Claude models - Cross-region inference model ID handling - Multiple AWS credential sources (profile, IAM keys, API keys) - Image support in messages and tool results - Unicode surrogate sanitization Also adds 'Adding a New Provider' documentation to AGENTS.md and README. Co-authored-by: nickchan2 <nickchan2@users.noreply.github.com>
This commit is contained in:
Mario Zechner
parent
4f216d318f
commit
fd268479a4
31 changed files with 3550 additions and 2593 deletions
|
|
@ -4,6 +4,892 @@
|
|||
import type { Model } from "./types.js";
|
||||
|
||||
export const MODELS = {
|
||||
"amazon-bedrock": {
|
||||
"anthropic.claude-3-5-haiku-20241022-v1:0": {
|
||||
id: "anthropic.claude-3-5-haiku-20241022-v1:0",
|
||||
name: "Claude Haiku 3.5",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 0.8,
|
||||
output: 4,
|
||||
cacheRead: 0.08,
|
||||
cacheWrite: 1,
|
||||
},
|
||||
contextWindow: 200000,
|
||||
maxTokens: 8192,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"anthropic.claude-3-5-sonnet-20240620-v1:0": {
|
||||
id: "anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
name: "Claude Sonnet 3.5",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 3,
|
||||
output: 15,
|
||||
cacheRead: 0.3,
|
||||
cacheWrite: 3.75,
|
||||
},
|
||||
contextWindow: 200000,
|
||||
maxTokens: 8192,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"anthropic.claude-3-5-sonnet-20241022-v2:0": {
|
||||
id: "anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
name: "Claude Sonnet 3.5 v2",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 3,
|
||||
output: 15,
|
||||
cacheRead: 0.3,
|
||||
cacheWrite: 3.75,
|
||||
},
|
||||
contextWindow: 200000,
|
||||
maxTokens: 8192,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"anthropic.claude-3-haiku-20240307-v1:0": {
|
||||
id: "anthropic.claude-3-haiku-20240307-v1:0",
|
||||
name: "Claude Haiku 3",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 0.25,
|
||||
output: 1.25,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 200000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"anthropic.claude-3-opus-20240229-v1:0": {
|
||||
id: "anthropic.claude-3-opus-20240229-v1:0",
|
||||
name: "Claude Opus 3",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 15,
|
||||
output: 75,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 200000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"anthropic.claude-3-sonnet-20240229-v1:0": {
|
||||
id: "anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
name: "Claude Sonnet 3",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 3,
|
||||
output: 15,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 200000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"cohere.command-r-plus-v1:0": {
|
||||
id: "cohere.command-r-plus-v1:0",
|
||||
name: "Command R+",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 3,
|
||||
output: 15,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"cohere.command-r-v1:0": {
|
||||
id: "cohere.command-r-v1:0",
|
||||
name: "Command R",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.5,
|
||||
output: 1.5,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"deepseek.v3-v1:0": {
|
||||
id: "deepseek.v3-v1:0",
|
||||
name: "DeepSeek-V3.1",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.58,
|
||||
output: 1.68,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 163840,
|
||||
maxTokens: 81920,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"global.amazon.nova-2-lite-v1:0": {
|
||||
id: "global.amazon.nova-2-lite-v1:0",
|
||||
name: "Nova 2 Lite",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 0.33,
|
||||
output: 2.75,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"global.anthropic.claude-haiku-4-5-20251001-v1:0": {
|
||||
id: "global.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
name: "Claude Haiku 4.5",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 1,
|
||||
output: 5,
|
||||
cacheRead: 0.1,
|
||||
cacheWrite: 1.25,
|
||||
},
|
||||
contextWindow: 200000,
|
||||
maxTokens: 64000,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"global.anthropic.claude-opus-4-5-20251101-v1:0": {
|
||||
id: "global.anthropic.claude-opus-4-5-20251101-v1:0",
|
||||
name: "Claude Opus 4.5",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 5,
|
||||
output: 25,
|
||||
cacheRead: 0.5,
|
||||
cacheWrite: 6.25,
|
||||
},
|
||||
contextWindow: 200000,
|
||||
maxTokens: 64000,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"global.anthropic.claude-sonnet-4-20250514-v1:0": {
|
||||
id: "global.anthropic.claude-sonnet-4-20250514-v1:0",
|
||||
name: "Claude Sonnet 4",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 3,
|
||||
output: 15,
|
||||
cacheRead: 0.3,
|
||||
cacheWrite: 3.75,
|
||||
},
|
||||
contextWindow: 200000,
|
||||
maxTokens: 64000,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"global.anthropic.claude-sonnet-4-5-20250929-v1:0": {
|
||||
id: "global.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
name: "Claude Sonnet 4.5",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 3,
|
||||
output: 15,
|
||||
cacheRead: 0.3,
|
||||
cacheWrite: 3.75,
|
||||
},
|
||||
contextWindow: 200000,
|
||||
maxTokens: 64000,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"google.gemma-3-27b-it": {
|
||||
id: "google.gemma-3-27b-it",
|
||||
name: "Google Gemma 3 27B Instruct",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 0.12,
|
||||
output: 0.2,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 202752,
|
||||
maxTokens: 8192,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"google.gemma-3-4b-it": {
|
||||
id: "google.gemma-3-4b-it",
|
||||
name: "Gemma 3 4B IT",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 0.04,
|
||||
output: 0.08,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"meta.llama3-1-70b-instruct-v1:0": {
|
||||
id: "meta.llama3-1-70b-instruct-v1:0",
|
||||
name: "Llama 3.1 70B Instruct",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.72,
|
||||
output: 0.72,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"meta.llama3-1-8b-instruct-v1:0": {
|
||||
id: "meta.llama3-1-8b-instruct-v1:0",
|
||||
name: "Llama 3.1 8B Instruct",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.22,
|
||||
output: 0.22,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"minimax.minimax-m2": {
|
||||
id: "minimax.minimax-m2",
|
||||
name: "MiniMax M2",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.3,
|
||||
output: 1.2,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 204608,
|
||||
maxTokens: 128000,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"mistral.ministral-3-14b-instruct": {
|
||||
id: "mistral.ministral-3-14b-instruct",
|
||||
name: "Ministral 14B 3.0",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.2,
|
||||
output: 0.2,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"mistral.ministral-3-8b-instruct": {
|
||||
id: "mistral.ministral-3-8b-instruct",
|
||||
name: "Ministral 3 8B",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.15,
|
||||
output: 0.15,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"mistral.mistral-large-2402-v1:0": {
|
||||
id: "mistral.mistral-large-2402-v1:0",
|
||||
name: "Mistral Large (24.02)",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.5,
|
||||
output: 1.5,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"mistral.voxtral-mini-3b-2507": {
|
||||
id: "mistral.voxtral-mini-3b-2507",
|
||||
name: "Voxtral Mini 3B 2507",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.04,
|
||||
output: 0.04,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"mistral.voxtral-small-24b-2507": {
|
||||
id: "mistral.voxtral-small-24b-2507",
|
||||
name: "Voxtral Small 24B 2507",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.15,
|
||||
output: 0.35,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 32000,
|
||||
maxTokens: 8192,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"moonshot.kimi-k2-thinking": {
|
||||
id: "moonshot.kimi-k2-thinking",
|
||||
name: "Kimi K2 Thinking",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.6,
|
||||
output: 2.5,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 256000,
|
||||
maxTokens: 256000,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"nvidia.nemotron-nano-12b-v2": {
|
||||
id: "nvidia.nemotron-nano-12b-v2",
|
||||
name: "NVIDIA Nemotron Nano 12B v2 VL BF16",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 0.2,
|
||||
output: 0.6,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"nvidia.nemotron-nano-9b-v2": {
|
||||
id: "nvidia.nemotron-nano-9b-v2",
|
||||
name: "NVIDIA Nemotron Nano 9B v2",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.06,
|
||||
output: 0.23,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"openai.gpt-oss-120b-1:0": {
|
||||
id: "openai.gpt-oss-120b-1:0",
|
||||
name: "gpt-oss-120b",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.15,
|
||||
output: 0.6,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"openai.gpt-oss-20b-1:0": {
|
||||
id: "openai.gpt-oss-20b-1:0",
|
||||
name: "gpt-oss-20b",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.07,
|
||||
output: 0.3,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"openai.gpt-oss-safeguard-120b": {
|
||||
id: "openai.gpt-oss-safeguard-120b",
|
||||
name: "GPT OSS Safeguard 120B",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.15,
|
||||
output: 0.6,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"openai.gpt-oss-safeguard-20b": {
|
||||
id: "openai.gpt-oss-safeguard-20b",
|
||||
name: "GPT OSS Safeguard 20B",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.07,
|
||||
output: 0.2,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"qwen.qwen3-235b-a22b-2507-v1:0": {
|
||||
id: "qwen.qwen3-235b-a22b-2507-v1:0",
|
||||
name: "Qwen3 235B A22B 2507",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.22,
|
||||
output: 0.88,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 262144,
|
||||
maxTokens: 131072,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"qwen.qwen3-32b-v1:0": {
|
||||
id: "qwen.qwen3-32b-v1:0",
|
||||
name: "Qwen3 32B (dense)",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.15,
|
||||
output: 0.6,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 16384,
|
||||
maxTokens: 16384,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"qwen.qwen3-coder-30b-a3b-v1:0": {
|
||||
id: "qwen.qwen3-coder-30b-a3b-v1:0",
|
||||
name: "Qwen3 Coder 30B A3B Instruct",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.15,
|
||||
output: 0.6,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 262144,
|
||||
maxTokens: 131072,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"qwen.qwen3-coder-480b-a35b-v1:0": {
|
||||
id: "qwen.qwen3-coder-480b-a35b-v1:0",
|
||||
name: "Qwen3 Coder 480B A35B Instruct",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.22,
|
||||
output: 1.8,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 131072,
|
||||
maxTokens: 65536,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"qwen.qwen3-next-80b-a3b": {
|
||||
id: "qwen.qwen3-next-80b-a3b",
|
||||
name: "Qwen/Qwen3-Next-80B-A3B-Instruct",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.14,
|
||||
output: 1.4,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 262000,
|
||||
maxTokens: 262000,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"qwen.qwen3-vl-235b-a22b": {
|
||||
id: "qwen.qwen3-vl-235b-a22b",
|
||||
name: "Qwen/Qwen3-VL-235B-A22B-Instruct",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 0.3,
|
||||
output: 1.5,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 262000,
|
||||
maxTokens: 262000,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"us.amazon.nova-lite-v1:0": {
|
||||
id: "us.amazon.nova-lite-v1:0",
|
||||
name: "Nova Lite",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 0.06,
|
||||
output: 0.24,
|
||||
cacheRead: 0.015,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 300000,
|
||||
maxTokens: 8192,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"us.amazon.nova-micro-v1:0": {
|
||||
id: "us.amazon.nova-micro-v1:0",
|
||||
name: "Nova Micro",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.035,
|
||||
output: 0.14,
|
||||
cacheRead: 0.00875,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 8192,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"us.amazon.nova-premier-v1:0": {
|
||||
id: "us.amazon.nova-premier-v1:0",
|
||||
name: "Nova Premier",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 2.5,
|
||||
output: 12.5,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 1000000,
|
||||
maxTokens: 16384,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"us.amazon.nova-pro-v1:0": {
|
||||
id: "us.amazon.nova-pro-v1:0",
|
||||
name: "Nova Pro",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 0.8,
|
||||
output: 3.2,
|
||||
cacheRead: 0.2,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 300000,
|
||||
maxTokens: 8192,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"us.anthropic.claude-3-7-sonnet-20250219-v1:0": {
|
||||
id: "us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
name: "Claude Sonnet 3.7",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 3,
|
||||
output: 15,
|
||||
cacheRead: 0.3,
|
||||
cacheWrite: 3.75,
|
||||
},
|
||||
contextWindow: 200000,
|
||||
maxTokens: 8192,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"us.anthropic.claude-opus-4-1-20250805-v1:0": {
|
||||
id: "us.anthropic.claude-opus-4-1-20250805-v1:0",
|
||||
name: "Claude Opus 4.1",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 15,
|
||||
output: 75,
|
||||
cacheRead: 1.5,
|
||||
cacheWrite: 18.75,
|
||||
},
|
||||
contextWindow: 200000,
|
||||
maxTokens: 32000,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"us.anthropic.claude-opus-4-20250514-v1:0": {
|
||||
id: "us.anthropic.claude-opus-4-20250514-v1:0",
|
||||
name: "Claude Opus 4",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 15,
|
||||
output: 75,
|
||||
cacheRead: 1.5,
|
||||
cacheWrite: 18.75,
|
||||
},
|
||||
contextWindow: 200000,
|
||||
maxTokens: 32000,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"us.deepseek.r1-v1:0": {
|
||||
id: "us.deepseek.r1-v1:0",
|
||||
name: "DeepSeek-R1",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 1.35,
|
||||
output: 5.4,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 32768,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"us.meta.llama3-2-11b-instruct-v1:0": {
|
||||
id: "us.meta.llama3-2-11b-instruct-v1:0",
|
||||
name: "Llama 3.2 11B Instruct",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 0.16,
|
||||
output: 0.16,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"us.meta.llama3-2-1b-instruct-v1:0": {
|
||||
id: "us.meta.llama3-2-1b-instruct-v1:0",
|
||||
name: "Llama 3.2 1B Instruct",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.1,
|
||||
output: 0.1,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 131000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"us.meta.llama3-2-3b-instruct-v1:0": {
|
||||
id: "us.meta.llama3-2-3b-instruct-v1:0",
|
||||
name: "Llama 3.2 3B Instruct",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.15,
|
||||
output: 0.15,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 131000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"us.meta.llama3-2-90b-instruct-v1:0": {
|
||||
id: "us.meta.llama3-2-90b-instruct-v1:0",
|
||||
name: "Llama 3.2 90B Instruct",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 0.72,
|
||||
output: 0.72,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"us.meta.llama3-3-70b-instruct-v1:0": {
|
||||
id: "us.meta.llama3-3-70b-instruct-v1:0",
|
||||
name: "Llama 3.3 70B Instruct",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.72,
|
||||
output: 0.72,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128000,
|
||||
maxTokens: 4096,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"us.meta.llama4-maverick-17b-instruct-v1:0": {
|
||||
id: "us.meta.llama4-maverick-17b-instruct-v1:0",
|
||||
name: "Llama 4 Maverick 17B Instruct",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 0.24,
|
||||
output: 0.97,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 1000000,
|
||||
maxTokens: 16384,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
"us.meta.llama4-scout-17b-instruct-v1:0": {
|
||||
id: "us.meta.llama4-scout-17b-instruct-v1:0",
|
||||
name: "Llama 4 Scout 17B Instruct",
|
||||
api: "bedrock-converse-stream",
|
||||
provider: "amazon-bedrock",
|
||||
baseUrl: "https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 0.17,
|
||||
output: 0.66,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 3500000,
|
||||
maxTokens: 16384,
|
||||
} satisfies Model<"bedrock-converse-stream">,
|
||||
},
|
||||
"anthropic": {
|
||||
"claude-3-5-haiku-20241022": {
|
||||
id: "claude-3-5-haiku-20241022",
|
||||
|
|
@ -3660,7 +4546,7 @@ export const MODELS = {
|
|||
cacheWrite: 6.25,
|
||||
},
|
||||
contextWindow: 200000,
|
||||
maxTokens: 32000,
|
||||
maxTokens: 64000,
|
||||
} satisfies Model<"openai-completions">,
|
||||
"anthropic/claude-sonnet-4": {
|
||||
id: "anthropic/claude-sonnet-4",
|
||||
|
|
@ -3977,13 +4863,13 @@ export const MODELS = {
|
|||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0.39999999999999997,
|
||||
output: 1.75,
|
||||
input: 0.44999999999999996,
|
||||
output: 2.1500000000000004,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 163840,
|
||||
maxTokens: 65536,
|
||||
contextWindow: 131072,
|
||||
maxTokens: 32768,
|
||||
} satisfies Model<"openai-completions">,
|
||||
"deepseek/deepseek-r1-distill-llama-70b": {
|
||||
id: "deepseek/deepseek-r1-distill-llama-70b",
|
||||
|
|
|
|||
511
packages/ai/src/providers/amazon-bedrock.ts
Normal file
511
packages/ai/src/providers/amazon-bedrock.ts
Normal file
|
|
@ -0,0 +1,511 @@
|
|||
import {
|
||||
BedrockRuntimeClient,
|
||||
StopReason as BedrockStopReason,
|
||||
type Tool as BedrockTool,
|
||||
type ContentBlock,
|
||||
type ContentBlockDeltaEvent,
|
||||
type ContentBlockStartEvent,
|
||||
type ContentBlockStopEvent,
|
||||
ConversationRole,
|
||||
ConverseStreamCommand,
|
||||
type ConverseStreamMetadataEvent,
|
||||
ImageFormat,
|
||||
type Message,
|
||||
type ToolChoice,
|
||||
type ToolConfiguration,
|
||||
ToolResultStatus,
|
||||
} from "@aws-sdk/client-bedrock-runtime";
|
||||
|
||||
import { calculateCost } from "../models.js";
|
||||
import type {
|
||||
Api,
|
||||
AssistantMessage,
|
||||
Context,
|
||||
Model,
|
||||
StopReason,
|
||||
StreamFunction,
|
||||
StreamOptions,
|
||||
TextContent,
|
||||
ThinkingBudgets,
|
||||
ThinkingContent,
|
||||
ThinkingLevel,
|
||||
Tool,
|
||||
ToolCall,
|
||||
ToolResultMessage,
|
||||
} from "../types.js";
|
||||
import { AssistantMessageEventStream } from "../utils/event-stream.js";
|
||||
import { parseStreamingJson } from "../utils/json-parse.js";
|
||||
import { sanitizeSurrogates } from "../utils/sanitize-unicode.js";
|
||||
|
||||
export interface BedrockOptions extends StreamOptions {
|
||||
region?: string;
|
||||
profile?: string;
|
||||
toolChoice?: "auto" | "any" | "none" | { type: "tool"; name: string };
|
||||
/* See https://docs.aws.amazon.com/bedrock/latest/userguide/inference-reasoning.html for supported models. */
|
||||
reasoning?: ThinkingLevel;
|
||||
/* Custom token budgets per thinking level. Overrides default budgets. */
|
||||
thinkingBudgets?: ThinkingBudgets;
|
||||
/* Only supported by Claude 4.x models, see https://docs.aws.amazon.com/bedrock/latest/userguide/claude-messages-extended-thinking.html#claude-messages-extended-thinking-tool-use-interleaved */
|
||||
interleavedThinking?: boolean;
|
||||
}
|
||||
|
||||
type Block = (TextContent | ThinkingContent | ToolCall) & { index?: number; partialJson?: string };
|
||||
|
||||
export const streamBedrock: StreamFunction<"bedrock-converse-stream"> = (
|
||||
model: Model<"bedrock-converse-stream">,
|
||||
context: Context,
|
||||
options: BedrockOptions,
|
||||
): AssistantMessageEventStream => {
|
||||
const stream = new AssistantMessageEventStream();
|
||||
|
||||
(async () => {
|
||||
const output: AssistantMessage = {
|
||||
role: "assistant",
|
||||
content: [],
|
||||
api: "bedrock-converse-stream" as Api,
|
||||
provider: model.provider,
|
||||
model: model.id,
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
stopReason: "stop",
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
const blocks = output.content as Block[];
|
||||
|
||||
try {
|
||||
const client = new BedrockRuntimeClient({
|
||||
region: options.region || process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || "us-east-1",
|
||||
profile: options.profile,
|
||||
});
|
||||
|
||||
const command = new ConverseStreamCommand({
|
||||
modelId: model.id,
|
||||
messages: convertMessages(context),
|
||||
system: context.systemPrompt ? [{ text: sanitizeSurrogates(context.systemPrompt) }] : undefined,
|
||||
inferenceConfig: { maxTokens: options.maxTokens, temperature: options.temperature },
|
||||
toolConfig: convertToolConfig(context.tools, options.toolChoice),
|
||||
additionalModelRequestFields: buildAdditionalModelRequestFields(model, options),
|
||||
});
|
||||
|
||||
const response = await client.send(command, { abortSignal: options.signal });
|
||||
|
||||
for await (const item of response.stream!) {
|
||||
if (item.messageStart) {
|
||||
if (item.messageStart.role !== ConversationRole.ASSISTANT) {
|
||||
throw new Error("Unexpected assistant message start but got user message start instead");
|
||||
}
|
||||
stream.push({ type: "start", partial: output });
|
||||
} else if (item.contentBlockStart) {
|
||||
handleContentBlockStart(item.contentBlockStart, blocks, output, stream);
|
||||
} else if (item.contentBlockDelta) {
|
||||
handleContentBlockDelta(item.contentBlockDelta, blocks, output, stream);
|
||||
} else if (item.contentBlockStop) {
|
||||
handleContentBlockStop(item.contentBlockStop, blocks, output, stream);
|
||||
} else if (item.messageStop) {
|
||||
output.stopReason = mapStopReason(item.messageStop.stopReason);
|
||||
} else if (item.metadata) {
|
||||
handleMetadata(item.metadata, model, output);
|
||||
} else if (item.internalServerException) {
|
||||
throw new Error(`Internal server error: ${item.internalServerException.message}`);
|
||||
} else if (item.modelStreamErrorException) {
|
||||
throw new Error(`Model stream error: ${item.modelStreamErrorException.message}`);
|
||||
} else if (item.validationException) {
|
||||
throw new Error(`Validation error: ${item.validationException.message}`);
|
||||
} else if (item.throttlingException) {
|
||||
throw new Error(`Throttling error: ${item.throttlingException.message}`);
|
||||
} else if (item.serviceUnavailableException) {
|
||||
throw new Error(`Service unavailable: ${item.serviceUnavailableException.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (options.signal?.aborted) {
|
||||
throw new Error("Request was aborted");
|
||||
}
|
||||
|
||||
if (output.stopReason === "error" || output.stopReason === "aborted") {
|
||||
throw new Error("An unknown error occurred");
|
||||
}
|
||||
|
||||
stream.push({ type: "done", reason: output.stopReason, message: output });
|
||||
stream.end();
|
||||
} catch (error) {
|
||||
for (const block of output.content) {
|
||||
delete (block as Block).index;
|
||||
delete (block as Block).partialJson;
|
||||
}
|
||||
output.stopReason = options.signal?.aborted ? "aborted" : "error";
|
||||
output.errorMessage = error instanceof Error ? error.message : JSON.stringify(error);
|
||||
stream.push({ type: "error", reason: output.stopReason, error: output });
|
||||
stream.end();
|
||||
}
|
||||
})();
|
||||
|
||||
return stream;
|
||||
};
|
||||
|
||||
function handleContentBlockStart(
|
||||
event: ContentBlockStartEvent,
|
||||
blocks: Block[],
|
||||
output: AssistantMessage,
|
||||
stream: AssistantMessageEventStream,
|
||||
): void {
|
||||
const index = event.contentBlockIndex!;
|
||||
const start = event.start;
|
||||
|
||||
if (start?.toolUse) {
|
||||
const block: Block = {
|
||||
type: "toolCall",
|
||||
id: start.toolUse.toolUseId || "",
|
||||
name: start.toolUse.name || "",
|
||||
arguments: {},
|
||||
partialJson: "",
|
||||
index,
|
||||
};
|
||||
output.content.push(block);
|
||||
stream.push({ type: "toolcall_start", contentIndex: blocks.length - 1, partial: output });
|
||||
}
|
||||
}
|
||||
|
||||
function handleContentBlockDelta(
|
||||
event: ContentBlockDeltaEvent,
|
||||
blocks: Block[],
|
||||
output: AssistantMessage,
|
||||
stream: AssistantMessageEventStream,
|
||||
): void {
|
||||
const contentBlockIndex = event.contentBlockIndex!;
|
||||
const delta = event.delta;
|
||||
let index = blocks.findIndex((b) => b.index === contentBlockIndex);
|
||||
let block = blocks[index];
|
||||
|
||||
if (delta?.text !== undefined) {
|
||||
// If no text block exists yet, create one, as `handleContentBlockStart` is not sent for text blocks
|
||||
if (!block) {
|
||||
const newBlock: Block = { type: "text", text: "", index: contentBlockIndex };
|
||||
output.content.push(newBlock);
|
||||
index = blocks.length - 1;
|
||||
block = blocks[index];
|
||||
stream.push({ type: "text_start", contentIndex: index, partial: output });
|
||||
}
|
||||
if (block.type === "text") {
|
||||
block.text += delta.text;
|
||||
stream.push({ type: "text_delta", contentIndex: index, delta: delta.text, partial: output });
|
||||
}
|
||||
} else if (delta?.toolUse && block?.type === "toolCall") {
|
||||
block.partialJson = (block.partialJson || "") + (delta.toolUse.input || "");
|
||||
block.arguments = parseStreamingJson(block.partialJson);
|
||||
stream.push({ type: "toolcall_delta", contentIndex: index, delta: delta.toolUse.input || "", partial: output });
|
||||
} else if (delta?.reasoningContent) {
|
||||
let thinkingBlock = block;
|
||||
let thinkingIndex = index;
|
||||
|
||||
if (!thinkingBlock) {
|
||||
const newBlock: Block = { type: "thinking", thinking: "", thinkingSignature: "", index: contentBlockIndex };
|
||||
output.content.push(newBlock);
|
||||
thinkingIndex = blocks.length - 1;
|
||||
thinkingBlock = blocks[thinkingIndex];
|
||||
stream.push({ type: "thinking_start", contentIndex: thinkingIndex, partial: output });
|
||||
}
|
||||
|
||||
if (thinkingBlock?.type === "thinking") {
|
||||
if (delta.reasoningContent.text) {
|
||||
thinkingBlock.thinking += delta.reasoningContent.text;
|
||||
stream.push({
|
||||
type: "thinking_delta",
|
||||
contentIndex: thinkingIndex,
|
||||
delta: delta.reasoningContent.text,
|
||||
partial: output,
|
||||
});
|
||||
}
|
||||
if (delta.reasoningContent.signature) {
|
||||
thinkingBlock.thinkingSignature =
|
||||
(thinkingBlock.thinkingSignature || "") + delta.reasoningContent.signature;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function handleMetadata(
|
||||
event: ConverseStreamMetadataEvent,
|
||||
model: Model<"bedrock-converse-stream">,
|
||||
output: AssistantMessage,
|
||||
): void {
|
||||
if (event.usage) {
|
||||
output.usage.input = event.usage.inputTokens || 0;
|
||||
output.usage.output = event.usage.outputTokens || 0;
|
||||
output.usage.cacheRead = event.usage.cacheReadInputTokens || 0;
|
||||
output.usage.cacheWrite = event.usage.cacheWriteInputTokens || 0;
|
||||
output.usage.totalTokens = event.usage.totalTokens || output.usage.input + output.usage.output;
|
||||
calculateCost(model, output.usage);
|
||||
}
|
||||
}
|
||||
|
||||
function handleContentBlockStop(
|
||||
event: ContentBlockStopEvent,
|
||||
blocks: Block[],
|
||||
output: AssistantMessage,
|
||||
stream: AssistantMessageEventStream,
|
||||
): void {
|
||||
const index = blocks.findIndex((b) => b.index === event.contentBlockIndex);
|
||||
const block = blocks[index];
|
||||
if (!block) return;
|
||||
delete (block as Block).index;
|
||||
|
||||
switch (block.type) {
|
||||
case "text":
|
||||
stream.push({ type: "text_end", contentIndex: index, content: block.text, partial: output });
|
||||
break;
|
||||
case "thinking":
|
||||
stream.push({ type: "thinking_end", contentIndex: index, content: block.thinking, partial: output });
|
||||
break;
|
||||
case "toolCall":
|
||||
block.arguments = parseStreamingJson(block.partialJson);
|
||||
delete (block as Block).partialJson;
|
||||
stream.push({ type: "toolcall_end", contentIndex: index, toolCall: block, partial: output });
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
function convertMessages(context: Context): Message[] {
|
||||
const result: Message[] = [];
|
||||
const messages = context.messages;
|
||||
|
||||
for (let i = 0; i < messages.length; i++) {
|
||||
const m = messages[i];
|
||||
|
||||
switch (m.role) {
|
||||
case "user":
|
||||
result.push({
|
||||
role: ConversationRole.USER,
|
||||
content:
|
||||
typeof m.content === "string"
|
||||
? [{ text: sanitizeSurrogates(m.content) }]
|
||||
: m.content.map((c) => {
|
||||
switch (c.type) {
|
||||
case "text":
|
||||
return { text: sanitizeSurrogates(c.text) };
|
||||
case "image":
|
||||
return { image: createImageBlock(c.mimeType, c.data) };
|
||||
default:
|
||||
throw new Error("Unknown user content type");
|
||||
}
|
||||
}),
|
||||
});
|
||||
break;
|
||||
case "assistant": {
|
||||
// Skip assistant messages with empty content (e.g., from aborted requests)
|
||||
// Bedrock rejects messages with empty content arrays
|
||||
if (m.content.length === 0) {
|
||||
continue;
|
||||
}
|
||||
const contentBlocks: ContentBlock[] = [];
|
||||
for (const c of m.content) {
|
||||
switch (c.type) {
|
||||
case "text":
|
||||
// Skip empty text blocks
|
||||
if (c.text.trim().length === 0) continue;
|
||||
contentBlocks.push({ text: sanitizeSurrogates(c.text) });
|
||||
break;
|
||||
case "toolCall":
|
||||
contentBlocks.push({
|
||||
toolUse: { toolUseId: c.id, name: c.name, input: c.arguments },
|
||||
});
|
||||
break;
|
||||
case "thinking":
|
||||
// Skip empty thinking blocks
|
||||
if (c.thinking.trim().length === 0) continue;
|
||||
contentBlocks.push({
|
||||
reasoningContent: {
|
||||
reasoningText: { text: sanitizeSurrogates(c.thinking), signature: c.thinkingSignature },
|
||||
},
|
||||
});
|
||||
break;
|
||||
default:
|
||||
throw new Error("Unknown assistant content type");
|
||||
}
|
||||
}
|
||||
// Skip if all content blocks were filtered out
|
||||
if (contentBlocks.length === 0) {
|
||||
continue;
|
||||
}
|
||||
result.push({
|
||||
role: ConversationRole.ASSISTANT,
|
||||
content: contentBlocks,
|
||||
});
|
||||
break;
|
||||
}
|
||||
case "toolResult": {
|
||||
// Collect all consecutive toolResult messages into a single user message
|
||||
// Bedrock requires all tool results to be in one message
|
||||
const toolResults: ContentBlock.ToolResultMember[] = [];
|
||||
|
||||
// Add current tool result
|
||||
for (const c of m.content) {
|
||||
toolResults.push({
|
||||
toolResult: {
|
||||
toolUseId: m.toolCallId,
|
||||
content: [
|
||||
c.type === "image"
|
||||
? { image: createImageBlock(c.mimeType, c.data) }
|
||||
: { text: sanitizeSurrogates(c.text) },
|
||||
],
|
||||
status: m.isError ? ToolResultStatus.ERROR : ToolResultStatus.SUCCESS,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Look ahead for consecutive toolResult messages
|
||||
let j = i + 1;
|
||||
while (j < messages.length && messages[j].role === "toolResult") {
|
||||
const nextMsg = messages[j] as ToolResultMessage;
|
||||
for (const c of nextMsg.content) {
|
||||
toolResults.push({
|
||||
toolResult: {
|
||||
toolUseId: nextMsg.toolCallId,
|
||||
content: [
|
||||
c.type === "image"
|
||||
? { image: createImageBlock(c.mimeType, c.data) }
|
||||
: { text: sanitizeSurrogates(c.text) },
|
||||
],
|
||||
status: nextMsg.isError ? ToolResultStatus.ERROR : ToolResultStatus.SUCCESS,
|
||||
},
|
||||
});
|
||||
}
|
||||
j++;
|
||||
}
|
||||
|
||||
// Skip the messages we've already processed
|
||||
i = j - 1;
|
||||
|
||||
result.push({
|
||||
role: ConversationRole.USER,
|
||||
content: toolResults,
|
||||
});
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new Error("Unknown message role");
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function convertToolConfig(
|
||||
tools: Tool[] | undefined,
|
||||
toolChoice: BedrockOptions["toolChoice"],
|
||||
): ToolConfiguration | undefined {
|
||||
if (!tools?.length || toolChoice === "none") return undefined;
|
||||
|
||||
const bedrockTools: BedrockTool[] = tools.map((tool) => ({
|
||||
toolSpec: {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
inputSchema: { json: tool.parameters },
|
||||
},
|
||||
}));
|
||||
|
||||
let bedrockToolChoice: ToolChoice | undefined;
|
||||
switch (toolChoice) {
|
||||
case "auto":
|
||||
bedrockToolChoice = { auto: {} };
|
||||
break;
|
||||
case "any":
|
||||
bedrockToolChoice = { any: {} };
|
||||
break;
|
||||
default:
|
||||
if (toolChoice?.type === "tool") {
|
||||
bedrockToolChoice = { tool: { name: toolChoice.name } };
|
||||
}
|
||||
}
|
||||
|
||||
return { tools: bedrockTools, toolChoice: bedrockToolChoice };
|
||||
}
|
||||
|
||||
function mapStopReason(reason: string | undefined): StopReason {
|
||||
switch (reason) {
|
||||
case BedrockStopReason.END_TURN:
|
||||
case BedrockStopReason.STOP_SEQUENCE:
|
||||
return "stop";
|
||||
case BedrockStopReason.MAX_TOKENS:
|
||||
case BedrockStopReason.MODEL_CONTEXT_WINDOW_EXCEEDED:
|
||||
return "length";
|
||||
case BedrockStopReason.TOOL_USE:
|
||||
return "toolUse";
|
||||
default:
|
||||
return "error";
|
||||
}
|
||||
}
|
||||
|
||||
function buildAdditionalModelRequestFields(
|
||||
model: Model<"bedrock-converse-stream">,
|
||||
options: BedrockOptions,
|
||||
): Record<string, any> | undefined {
|
||||
if (!options.reasoning || !model.reasoning) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (model.id.includes("anthropic.claude")) {
|
||||
const defaultBudgets: Record<ThinkingLevel, number> = {
|
||||
minimal: 1024,
|
||||
low: 2048,
|
||||
medium: 8192,
|
||||
high: 16384,
|
||||
xhigh: 16384, // Claude doesn't support xhigh, clamp to high
|
||||
};
|
||||
|
||||
// Custom budgets override defaults (xhigh not in ThinkingBudgets, use high)
|
||||
const level = options.reasoning === "xhigh" ? "high" : options.reasoning;
|
||||
const budget = options.thinkingBudgets?.[level] ?? defaultBudgets[options.reasoning];
|
||||
|
||||
const result: Record<string, any> = {
|
||||
thinking: {
|
||||
type: "enabled",
|
||||
budget_tokens: budget,
|
||||
},
|
||||
};
|
||||
|
||||
if (options.interleavedThinking) {
|
||||
result.anthropic_beta = ["interleaved-thinking-2025-05-14"];
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function createImageBlock(mimeType: string, data: string) {
|
||||
let format: ImageFormat;
|
||||
switch (mimeType) {
|
||||
case "image/jpeg":
|
||||
case "image/jpg":
|
||||
format = ImageFormat.JPEG;
|
||||
break;
|
||||
case "image/png":
|
||||
format = ImageFormat.PNG;
|
||||
break;
|
||||
case "image/gif":
|
||||
format = ImageFormat.GIF;
|
||||
break;
|
||||
case "image/webp":
|
||||
format = ImageFormat.WEBP;
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unknown image type: ${mimeType}`);
|
||||
}
|
||||
|
||||
const binaryString = atob(data);
|
||||
const bytes = new Uint8Array(binaryString.length);
|
||||
for (let i = 0; i < binaryString.length; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
}
|
||||
|
||||
return { source: { bytes }, format };
|
||||
}
|
||||
|
|
@ -287,7 +287,7 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = (
|
|||
}
|
||||
|
||||
if (output.stopReason === "aborted" || output.stopReason === "error") {
|
||||
throw new Error("An unkown error ocurred");
|
||||
throw new Error("An unknown error occurred");
|
||||
}
|
||||
|
||||
stream.push({ type: "done", reason: output.stopReason, message: output });
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { existsSync } from "node:fs";
|
|||
import { homedir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { supportsXhigh } from "./models.js";
|
||||
import { type BedrockOptions, streamBedrock } from "./providers/amazon-bedrock.js";
|
||||
import { type AnthropicOptions, streamAnthropic } from "./providers/anthropic.js";
|
||||
import { type GoogleOptions, streamGoogle } from "./providers/google.js";
|
||||
import {
|
||||
|
|
@ -74,6 +75,20 @@ export function getEnvApiKey(provider: any): string | undefined {
|
|||
}
|
||||
}
|
||||
|
||||
if (provider === "amazon-bedrock") {
|
||||
// Amazon Bedrock supports multiple credential sources:
|
||||
// 1. AWS_PROFILE - named profile from ~/.aws/credentials
|
||||
// 2. AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY - standard IAM keys
|
||||
// 3. AWS_BEARER_TOKEN_BEDROCK - Bedrock API keys (bearer token)
|
||||
if (
|
||||
process.env.AWS_PROFILE ||
|
||||
(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) ||
|
||||
process.env.AWS_BEARER_TOKEN_BEDROCK
|
||||
) {
|
||||
return "<authenticated>";
|
||||
}
|
||||
}
|
||||
|
||||
const envMap: Record<string, string> = {
|
||||
openai: "OPENAI_API_KEY",
|
||||
google: "GEMINI_API_KEY",
|
||||
|
|
@ -98,6 +113,9 @@ export function stream<TApi extends Api>(
|
|||
// Vertex AI uses Application Default Credentials, not API keys
|
||||
if (model.api === "google-vertex") {
|
||||
return streamGoogleVertex(model as Model<"google-vertex">, context, options as GoogleVertexOptions);
|
||||
} else if (model.api === "bedrock-converse-stream") {
|
||||
// Bedrock doesn't have any API keys instead it sources credentials from standard AWS env variables or from given AWS profile.
|
||||
return streamBedrock(model as Model<"bedrock-converse-stream">, context, (options || {}) as BedrockOptions);
|
||||
}
|
||||
|
||||
const apiKey = options?.apiKey || getEnvApiKey(model.provider);
|
||||
|
|
@ -156,6 +174,10 @@ export function streamSimple<TApi extends Api>(
|
|||
if (model.api === "google-vertex") {
|
||||
const providerOptions = mapOptionsForApi(model, options, undefined);
|
||||
return stream(model, context, providerOptions);
|
||||
} else if (model.api === "bedrock-converse-stream") {
|
||||
// Bedrock doesn't have any API keys instead it sources credentials from standard AWS env variables or from given AWS profile.
|
||||
const providerOptions = mapOptionsForApi(model, options, undefined);
|
||||
return stream(model, context, providerOptions);
|
||||
}
|
||||
|
||||
const apiKey = options?.apiKey || getEnvApiKey(model.provider);
|
||||
|
|
@ -228,6 +250,13 @@ function mapOptionsForApi<TApi extends Api>(
|
|||
} satisfies AnthropicOptions;
|
||||
}
|
||||
|
||||
case "bedrock-converse-stream":
|
||||
return {
|
||||
...base,
|
||||
reasoning: options?.reasoning,
|
||||
thinkingBudgets: options?.thinkingBudgets,
|
||||
} satisfies BedrockOptions;
|
||||
|
||||
case "openai-completions":
|
||||
return {
|
||||
...base,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import type { BedrockOptions } from "./providers/amazon-bedrock.js";
|
||||
import type { AnthropicOptions } from "./providers/anthropic.js";
|
||||
import type { GoogleOptions } from "./providers/google.js";
|
||||
import type { GoogleGeminiCliOptions } from "./providers/google-gemini-cli.js";
|
||||
|
|
@ -14,12 +15,14 @@ export type Api =
|
|||
| "openai-responses"
|
||||
| "openai-codex-responses"
|
||||
| "anthropic-messages"
|
||||
| "bedrock-converse-stream"
|
||||
| "google-generative-ai"
|
||||
| "google-gemini-cli"
|
||||
| "google-vertex";
|
||||
|
||||
export interface ApiOptionsMap {
|
||||
"anthropic-messages": AnthropicOptions;
|
||||
"bedrock-converse-stream": BedrockOptions;
|
||||
"openai-completions": OpenAICompletionsOptions;
|
||||
"openai-responses": OpenAIResponsesOptions;
|
||||
"openai-codex-responses": OpenAICodexResponsesOptions;
|
||||
|
|
@ -40,6 +43,7 @@ const _exhaustive: _CheckExhaustive = true;
|
|||
export type OptionsForApi<TApi extends Api> = ApiOptionsMap[TApi];
|
||||
|
||||
export type KnownProvider =
|
||||
| "amazon-bedrock"
|
||||
| "anthropic"
|
||||
| "google"
|
||||
| "google-gemini-cli"
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import type { AssistantMessage } from "../types.js";
|
|||
*/
|
||||
const OVERFLOW_PATTERNS = [
|
||||
/prompt is too long/i, // Anthropic
|
||||
/input is too long for requested model/i, // Amazon Bedrock
|
||||
/exceeds the context window/i, // OpenAI (Completions & Responses API)
|
||||
/input token count.*exceeds the maximum/i, // Google (Gemini)
|
||||
/maximum prompt length is \d+/i, // xAI (Grok)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue