Add image support in tool results across all providers

Tool results now use content blocks and can include both text and images.
All providers (Anthropic, Google, OpenAI Completions, OpenAI Responses)
correctly pass images from tool results to LLMs.

- Update ToolResultMessage type to use content blocks
- Add placeholder text for image-only tool results in Google/Anthropic
- OpenAI providers send tool result + follow-up user message with images
- Fix Anthropic JSON parsing for empty tool arguments
- Add comprehensive tests for image-only and text+image tool results
- Update README with tool result content blocks API
This commit is contained in:
Mario Zechner 2025-11-12 10:45:56 +01:00
parent 9dac37d836
commit 84dcab219b
37 changed files with 720 additions and 544 deletions

View file

@ -16,7 +16,11 @@ export class BashRenderer implements ToolRenderer<BashParams, undefined> {
// With result: show command + output
if (result && params?.command) {
const output = result.output || "";
const output =
result.content
?.filter((c) => c.type === "text")
.map((c: any) => c.text)
.join("\n") || "";
const combined = output ? `> ${params.command}\n\n${output}` : `> ${params.command}`;
return {
content: html`

View file

@ -16,7 +16,11 @@ export class CalculateRenderer implements ToolRenderer<CalculateParams, undefine
// Full params + full result
if (result && params?.expression) {
const output = result.output || "";
const output =
result.content
?.filter((c) => c.type === "text")
.map((c: any) => c.text)
.join("\n") || "";
// Error: show expression in header, error below
if (result.isError) {

View file

@ -25,7 +25,11 @@ export class DefaultRenderer implements ToolRenderer {
// With result: show header + params + result
if (result) {
let outputJson = result.output || i18n("(no output)");
let outputJson =
result.content
?.filter((c) => c.type === "text")
.map((c: any) => c.text)
.join("\n") || i18n("(no output)");
let outputLanguage = "text";
// Try to parse and pretty-print if it's valid JSON

View file

@ -19,7 +19,11 @@ export class GetCurrentTimeRenderer implements ToolRenderer<GetCurrentTimeParams
// Full params + full result
if (result && params) {
const output = result.output || "";
const output =
result.content
?.filter((c) => c.type === "text")
.map((c: any) => c.text)
.join("\n") || "";
const headerText = params.timezone
? `${i18n("Getting current time in")} ${params.timezone}`
: i18n("Getting current date and time");
@ -43,7 +47,11 @@ export class GetCurrentTimeRenderer implements ToolRenderer<GetCurrentTimeParams
// Full result, no params
if (result) {
const output = result.output || "";
const output =
result.content
?.filter((c) => c.type === "text")
.map((c: any) => c.text)
.join("\n") || "";
// Error: show header, error below
if (result.isError) {