getcompanion-ai/co-mono
1
0
Fork 0
mirror of https://github.com/getcompanion-ai/co-mono.git synced 2026-04-15 06:04:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(coding-agent): forward images through steer/followUp during streaming

Browse source 
prompt() computed currentImages but never passed them to _queueSteer()
or _queueFollowUp() in the streaming branch. Both methods only accepted
text and built content as [{ type: 'text', text }], dropping images.

- _queueSteer/_queueFollowUp now accept optional ImageContent[]
- streaming branch in prompt() passes currentImages through
- public steer()/followUp() accept and forward optional images
- RPC types, handler, and client updated for steer/follow_up images
- rpc.md: document images on steer/follow_up, fix ImageContent examples
This commit is contained in:
Aliou Diallo 2026-02-05 02:33:05 +01:00
parent 634899aba0
commit b315abf998
5 changed files with 44 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

18
packages/coding-agent/docs/rpc.md
View file

@ -38,7 +38,7 @@ Send a user prompt to the agent. Returns immediately; events stream asynchronous
With images:
```json
{"type": "prompt", "message": "What's in this image?", "images": [{"type": "image", "source": {"type": "base64", "mediaType": "image/png", "data": "..."}}]}
{"type": "prompt", "message": "What's in this image?", "images": [{"type": "image", "data": "base64-encoded-data", "mimeType": "image/png"}]}
```
**During streaming**: If the agent is already streaming, you must specify `streamingBehavior` to queue the message:
@ -61,7 +61,7 @@ Response:
{"id": "req-1", "type": "response", "command": "prompt", "success": true}
```
The `images` field is optional. Each image uses `ImageContent` format with base64 or URL source.
The `images` field is optional. Each image uses `ImageContent` format: `{"type": "image", "data": "base64-encoded-data", "mimeType": "image/png"}`.
#### steer
@ -71,6 +71,13 @@ Queue a steering message to interrupt the agent mid-run. Delivered after current
{"type": "steer", "message": "Stop and do this instead"}
```
With images:
```json
{"type": "steer", "message": "Look at this instead", "images": [{"type": "image", "data": "base64-encoded-data", "mimeType": "image/png"}]}
```
The `images` field is optional. Each image uses `ImageContent` format (same as `prompt`).
Response:
```json
{"type": "response", "command": "steer", "success": true}
@ -86,6 +93,13 @@ Queue a follow-up message to be processed after the agent finishes. Delivered on
{"type": "follow_up", "message": "After you're done, also do this"}
```
With images:
```json
{"type": "follow_up", "message": "Also check this image", "images": [{"type": "image", "data": "base64-encoded-data", "mimeType": "image/png"}]}
```
The `images` field is optional. Each image uses `ImageContent` format (same as `prompt`).
Response:
```json
{"type": "response", "command": "follow_up", "success": true}

30
packages/coding-agent/src/core/agent-session.ts
View file

@ -695,9 +695,9 @@ export class AgentSession {
);
}
if (options.streamingBehavior === "followUp") {
await this._queueFollowUp(expandedText);
await this._queueFollowUp(expandedText, currentImages);
} else {
await this._queueSteer(expandedText);
await this._queueSteer(expandedText, currentImages);
}
return;
}
@ -856,9 +856,10 @@ export class AgentSession {
* Queue a steering message to interrupt the agent mid-run.
* Delivered after current tool execution, skips remaining tools.
* Expands skill commands and prompt templates. Errors on extension commands.
* @param images Optional image attachments to include with the message
* @throws Error if text is an extension command
*/
async steer(text: string): Promise<void> {
async steer(text: string, images?: ImageContent[]): Promise<void> {
// Check for extension commands (cannot be queued)
if (text.startsWith("/")) {
this._throwIfExtensionCommand(text);
@ -868,16 +869,17 @@ export class AgentSession {
let expandedText = this._expandSkillCommand(text);
expandedText = expandPromptTemplate(expandedText, [...this.promptTemplates]);
await this._queueSteer(expandedText);
await this._queueSteer(expandedText, images);
}
/**
* Queue a follow-up message to be processed after the agent finishes.
* Delivered only when agent has no more tool calls or steering messages.
* Expands skill commands and prompt templates. Errors on extension commands.
* @param images Optional image attachments to include with the message
* @throws Error if text is an extension command
*/
async followUp(text: string): Promise<void> {
async followUp(text: string, images?: ImageContent[]): Promise<void> {
// Check for extension commands (cannot be queued)
if (text.startsWith("/")) {
this._throwIfExtensionCommand(text);
@ -887,17 +889,21 @@ export class AgentSession {
let expandedText = this._expandSkillCommand(text);
expandedText = expandPromptTemplate(expandedText, [...this.promptTemplates]);
await this._queueFollowUp(expandedText);
await this._queueFollowUp(expandedText, images);
}
/**
* Internal: Queue a steering message (already expanded, no extension command check).
*/
private async _queueSteer(text: string): Promise<void> {
private async _queueSteer(text: string, images?: ImageContent[]): Promise<void> {
this._steeringMessages.push(text);
const content: (TextContent | ImageContent)[] = [{ type: "text", text }];
if (images) {
content.push(...images);
}
this.agent.steer({
role: "user",
content: [{ type: "text", text }],
content,
timestamp: Date.now(),
});
}
@ -905,11 +911,15 @@ export class AgentSession {
/**
* Internal: Queue a follow-up message (already expanded, no extension command check).
*/
private async _queueFollowUp(text: string): Promise<void> {
private async _queueFollowUp(text: string, images?: ImageContent[]): Promise<void> {
this._followUpMessages.push(text);
const content: (TextContent | ImageContent)[] = [{ type: "text", text }];
if (images) {
content.push(...images);
}
this.agent.followUp({
role: "user",
content: [{ type: "text", text }],
content,
timestamp: Date.now(),
});
}

8
packages/coding-agent/src/modes/rpc/rpc-client.ts
View file

@ -175,15 +175,15 @@ export class RpcClient {
/**
* Queue a steering message to interrupt the agent mid-run.
*/
async steer(message: string): Promise<void> {
await this.send({ type: "steer", message });
async steer(message: string, images?: ImageContent[]): Promise<void> {
await this.send({ type: "steer", message, images });
}
/**
* Queue a follow-up message to be processed after the agent finishes.
*/
async followUp(message: string): Promise<void> {
await this.send({ type: "follow_up", message });
async followUp(message: string, images?: ImageContent[]): Promise<void> {
await this.send({ type: "follow_up", message, images });
}
/**

4
packages/coding-agent/src/modes/rpc/rpc-mode.ts
View file

@ -328,12 +328,12 @@ export async function runRpcMode(session: AgentSession): Promise<never> {
}
case "steer": {
await session.steer(command.message);
await session.steer(command.message, command.images);
return success(id, "steer");
}
case "follow_up": {
await session.followUp(command.message);
await session.followUp(command.message, command.images);
return success(id, "follow_up");
}

4
packages/coding-agent/src/modes/rpc/rpc-types.ts
View file

@ -18,8 +18,8 @@ import type { CompactionResult } from "../../core/compaction/index.js";
export type RpcCommand =
// Prompting
| { id?: string; type: "prompt"; message: string; images?: ImageContent[]; streamingBehavior?: "steer" | "followUp" }
| { id?: string; type: "steer"; message: string }
| { id?: string; type: "follow_up"; message: string }
| { id?: string; type: "steer"; message: string; images?: ImageContent[] }
| { id?: string; type: "follow_up"; message: string; images?: ImageContent[] }
| { id?: string; type: "abort" }
| { id?: string; type: "new_session"; parentSession?: string }