getcompanion-ai/co-mono
1
0
Fork 0
mirror of https://github.com/getcompanion-ai/co-mono.git synced 2026-04-18 12:03:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add image support in tool results across all providers

Browse source 
Tool results now use content blocks and can include both text and images.
All providers (Anthropic, Google, OpenAI Completions, OpenAI Responses)
correctly pass images from tool results to LLMs.

- Update ToolResultMessage type to use content blocks
- Add placeholder text for image-only tool results in Google/Anthropic
- OpenAI providers send tool result + follow-up user message with images
- Fix Anthropic JSON parsing for empty tool arguments
- Add comprehensive tests for image-only and text+image tool results
- Update README with tool result content blocks API
This commit is contained in:
Mario Zechner 2025-11-12 10:45:56 +01:00
parent 9dac37d836
commit 84dcab219b
37 changed files with 720 additions and 544 deletions
Download patch file Download diff file Expand all files Collapse all files

25
packages/web-ui/src/tools/artifacts/artifacts-tool-renderer.ts
View file

@ -10,6 +10,17 @@ import type { ToolRenderer, ToolRenderResult } from "../types.js";
import { ArtifactPill } from "./ArtifactPill.js";
import type { ArtifactsPanel, ArtifactsParams } from "./artifacts.js";
// Helper to extract text from content blocks
function getTextOutput(result: ToolResultMessage<any> | undefined): string {
if (!result) return "";
return (
result.content
?.filter((c) => c.type === "text")
.map((c: any) => c.text)
.join("\n") || ""
);
}
// Helper to determine language for syntax highlighting
function getLanguageFromFilename(filename?: string): string {
if (!filename) return "text";
@ -109,8 +120,8 @@ export class ArtifactsToolRenderer implements ToolRenderer<ArtifactsParams, unde
${isDiff ? diffContent : content ? html`<code-block .code=${content} language=${getLanguageFromFilename(filename)}></code-block>` : ""}
${
isHtml
? html`<console-block .content=${result.output || i18n("An error occurred")} variant="error"></console-block>`
: html`<div class="text-sm text-destructive">${result.output || i18n("An error occurred")}</div>`
? html`<console-block .content=${getTextOutput(result) || i18n("An error occurred")} variant="error"></console-block>`
: html`<div class="text-sm text-destructive">${getTextOutput(result) || i18n("An error occurred")}</div>`
}
</div>
</div>
@ -124,7 +135,7 @@ export class ArtifactsToolRenderer implements ToolRenderer<ArtifactsParams, unde
content: html`
<div class="space-y-3">
${renderHeader(state, FileCode2, headerText)}
<div class="text-sm text-destructive">${result.output || i18n("An error occurred")}</div>
<div class="text-sm text-destructive">${getTextOutput(result) || i18n("An error occurred")}</div>
</div>
`,
isCustom: false,
@ -141,7 +152,7 @@ export class ArtifactsToolRenderer implements ToolRenderer<ArtifactsParams, unde
// GET command: show code block with file content
if (command === "get") {
const fileContent = result.output || i18n("(no output)");
const fileContent = getTextOutput(result) || i18n("(no output)");
return {
content: html`
<div>
@ -157,7 +168,7 @@ export class ArtifactsToolRenderer implements ToolRenderer<ArtifactsParams, unde
// LOGS command: show console block
if (command === "logs") {
const logs = result.output || i18n("(no output)");
const logs = getTextOutput(result) || i18n("(no output)");
return {
content: html`
<div>
@ -175,7 +186,7 @@ export class ArtifactsToolRenderer implements ToolRenderer<ArtifactsParams, unde
if (command === "create" || command === "rewrite") {
const codeContent = content || "";
const isHtml = filename?.endsWith(".html");
const logs = result.output || "";
const logs = getTextOutput(result) || "";
return {
content: html`
@ -193,7 +204,7 @@ export class ArtifactsToolRenderer implements ToolRenderer<ArtifactsParams, unde
if (command === "update") {
const isHtml = filename?.endsWith(".html");
const logs = result.output || "";
const logs = getTextOutput(result) || "";
return {
content: html`
<div>

2
packages/web-ui/src/tools/artifacts/artifacts.ts
View file

@ -284,7 +284,7 @@ export class ArtifactsPanel extends LitElement {
// Execute mutates our local store and returns a plain output
execute: async (_toolCallId: string, args: Static<typeof artifactsParamsSchema>, _signal?: AbortSignal) => {
const output = await this.executeCommand(args);
return { output, details: undefined };
return { content: [{ type: "text", text: output }], details: undefined };
},
};
}