mirror of
https://github.com/getcompanion-ai/co-mono.git
synced 2026-04-16 16:00:58 +00:00
feat(coding-agent): add blockImages setting to prevent image uploads
This commit is contained in:
parent
9063a71fe6
commit
b582a6b70d
9 changed files with 266 additions and 25 deletions
|
|
@ -2,6 +2,10 @@
|
|||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- Added `blockImages` setting to prevent images from being sent to LLM providers. Provides defense-in-depth blocking at Read tool, CLI file processor, AgentSession, and convertToLlm layer. ([#492](https://github.com/badlogic/pi-mono/pull/492))
|
||||
|
||||
## [0.37.2] - 2026-01-05
|
||||
|
||||
### Fixed
|
||||
|
|
|
|||
|
|
@ -18,6 +18,8 @@ export interface ProcessedFiles {
|
|||
export interface ProcessFileOptions {
|
||||
/** Whether to auto-resize images to 2000x2000 max. Default: true */
|
||||
autoResizeImages?: boolean;
|
||||
/** When true, skip image files with warning. Default: false */
|
||||
blockImages?: boolean;
|
||||
}
|
||||
|
||||
/** Process @file arguments into text content and image attachments */
|
||||
|
|
@ -48,6 +50,11 @@ export async function processFileArguments(fileArgs: string[], options?: Process
|
|||
const mimeType = await detectSupportedImageMimeTypeFromFile(absolutePath);
|
||||
|
||||
if (mimeType) {
|
||||
// Check if images are blocked
|
||||
if (options?.blockImages) {
|
||||
console.warn(chalk.yellow(`[blockImages] Skipping image file: ${absolutePath}`));
|
||||
continue;
|
||||
}
|
||||
// Handle image file
|
||||
const content = await readFile(absolutePath);
|
||||
const base64Content = content.toString("base64");
|
||||
|
|
|
|||
|
|
@ -592,7 +592,13 @@ export class AgentSession {
|
|||
// Add user message
|
||||
const userContent: (TextContent | ImageContent)[] = [{ type: "text", text: expandedText }];
|
||||
if (options?.images) {
|
||||
userContent.push(...options.images);
|
||||
const blockImages = this.settingsManager.getBlockImages();
|
||||
if (blockImages) {
|
||||
// Log warning for blocked images
|
||||
console.warn(`[blockImages] Blocked ${options.images.length} image(s) from being sent to provider`);
|
||||
} else {
|
||||
userContent.push(...options.images);
|
||||
}
|
||||
}
|
||||
messages.push({
|
||||
role: "user",
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@
|
|||
* ```
|
||||
*/
|
||||
|
||||
import { Agent, type AgentTool, type ThinkingLevel } from "@mariozechner/pi-agent-core";
|
||||
import type { Model } from "@mariozechner/pi-ai";
|
||||
import { Agent, type AgentMessage, type AgentTool, type ThinkingLevel } from "@mariozechner/pi-agent-core";
|
||||
import type { Message, Model } from "@mariozechner/pi-ai";
|
||||
import { join } from "path";
|
||||
import { getAgentDir } from "../config.js";
|
||||
import { AgentSession } from "./agent-session.js";
|
||||
|
|
@ -300,6 +300,7 @@ export function loadSettings(cwd?: string, agentDir?: string): Settings {
|
|||
extensions: manager.getExtensionPaths(),
|
||||
skills: manager.getSkillsSettings(),
|
||||
terminal: { showImages: manager.getShowImages() },
|
||||
images: { autoResize: manager.getImageAutoResize(), blockImages: manager.getBlockImages() },
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -425,8 +426,9 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {}
|
|||
time("discoverContextFiles");
|
||||
|
||||
const autoResizeImages = settingsManager.getImageAutoResize();
|
||||
const blockImages = settingsManager.getBlockImages();
|
||||
// Create ALL built-in tools for the registry (extensions can enable any of them)
|
||||
const allBuiltInToolsMap = createAllTools(cwd, { read: { autoResizeImages } });
|
||||
const allBuiltInToolsMap = createAllTools(cwd, { read: { autoResizeImages, blockImages } });
|
||||
// Determine initially active built-in tools (default: read, bash, edit, write)
|
||||
const defaultActiveToolNames: ToolName[] = ["read", "bash", "edit", "write"];
|
||||
const initialActiveToolNames: ToolName[] = options.tools
|
||||
|
|
@ -605,6 +607,31 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {}
|
|||
const promptTemplates = options.promptTemplates ?? discoverPromptTemplates(cwd, agentDir);
|
||||
time("discoverPromptTemplates");
|
||||
|
||||
// Create convertToLlm wrapper that filters images if blockImages is enabled
|
||||
const convertToLlmWithBlockImages = blockImages
|
||||
? (messages: AgentMessage[]): Message[] => {
|
||||
const converted = convertToLlm(messages);
|
||||
let totalFiltered = 0;
|
||||
// Filter out ImageContent from all messages as defense-in-depth
|
||||
const filtered = converted.map((msg) => {
|
||||
if (msg.role === "user" || msg.role === "toolResult") {
|
||||
const content = msg.content;
|
||||
if (Array.isArray(content)) {
|
||||
const originalLength = content.length;
|
||||
const filteredContent = content.filter((c) => c.type !== "image");
|
||||
totalFiltered += originalLength - filteredContent.length;
|
||||
return { ...msg, content: filteredContent };
|
||||
}
|
||||
}
|
||||
return msg;
|
||||
});
|
||||
if (totalFiltered > 0) {
|
||||
console.warn(`[blockImages] Defense-in-depth: filtered ${totalFiltered} image(s) at convertToLlm layer`);
|
||||
}
|
||||
return filtered;
|
||||
}
|
||||
: convertToLlm;
|
||||
|
||||
agent = new Agent({
|
||||
initialState: {
|
||||
systemPrompt,
|
||||
|
|
@ -612,7 +639,7 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {}
|
|||
thinkingLevel,
|
||||
tools: activeToolsArray,
|
||||
},
|
||||
convertToLlm,
|
||||
convertToLlm: convertToLlmWithBlockImages,
|
||||
transformContext: extensionRunner
|
||||
? async (messages) => {
|
||||
return extensionRunner.emitContext(messages);
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ export interface TerminalSettings {
|
|||
|
||||
export interface ImageSettings {
|
||||
autoResize?: boolean; // default: true (resize images to 2000x2000 max for better model compatibility)
|
||||
blockImages?: boolean; // default: false - when true, prevents all images from being sent to LLM providers
|
||||
}
|
||||
|
||||
export interface Settings {
|
||||
|
|
@ -398,6 +399,23 @@ export class SettingsManager {
|
|||
this.save();
|
||||
}
|
||||
|
||||
getBlockImages(): boolean {
|
||||
return this.settings.images?.blockImages ?? false;
|
||||
}
|
||||
|
||||
setBlockImages(blocked: boolean): void {
|
||||
if (!this.globalSettings.images) {
|
||||
this.globalSettings.images = {};
|
||||
}
|
||||
this.globalSettings.images.blockImages = blocked;
|
||||
// Also update active settings for inMemory mode
|
||||
if (!this.settings.images) {
|
||||
this.settings.images = {};
|
||||
}
|
||||
this.settings.images.blockImages = blocked;
|
||||
this.save();
|
||||
}
|
||||
|
||||
getEnabledModels(): string[] | undefined {
|
||||
return this.settings.enabledModels;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,10 +21,13 @@ export interface ReadToolDetails {
|
|||
export interface ReadToolOptions {
|
||||
/** Whether to auto-resize images to 2000x2000 max. Default: true */
|
||||
autoResizeImages?: boolean;
|
||||
/** When true, return text message instead of image content. Default: false */
|
||||
blockImages?: boolean;
|
||||
}
|
||||
|
||||
export function createReadTool(cwd: string, options?: ReadToolOptions): AgentTool<typeof readSchema> {
|
||||
const autoResizeImages = options?.autoResizeImages ?? true;
|
||||
const blockImages = options?.blockImages ?? false;
|
||||
return {
|
||||
name: "read",
|
||||
label: "read",
|
||||
|
|
@ -75,29 +78,39 @@ export function createReadTool(cwd: string, options?: ReadToolOptions): AgentToo
|
|||
let details: ReadToolDetails | undefined;
|
||||
|
||||
if (mimeType) {
|
||||
// Read as image (binary)
|
||||
const buffer = await readFile(absolutePath);
|
||||
const base64 = buffer.toString("base64");
|
||||
|
||||
if (autoResizeImages) {
|
||||
// Resize image if needed
|
||||
const resized = await resizeImage({ type: "image", data: base64, mimeType });
|
||||
const dimensionNote = formatDimensionNote(resized);
|
||||
|
||||
let textNote = `Read image file [${resized.mimeType}]`;
|
||||
if (dimensionNote) {
|
||||
textNote += `\n${dimensionNote}`;
|
||||
}
|
||||
|
||||
// Check if images are blocked
|
||||
if (blockImages) {
|
||||
content = [
|
||||
{ type: "text", text: textNote },
|
||||
{ type: "image", data: resized.data, mimeType: resized.mimeType },
|
||||
{
|
||||
type: "text",
|
||||
text: `[Image file detected: ${absolutePath}]\nImage reading is disabled. The 'blockImages' setting is enabled.`,
|
||||
},
|
||||
];
|
||||
} else {
|
||||
content = [
|
||||
{ type: "text", text: `Read image file [${mimeType}]` },
|
||||
{ type: "image", data: base64, mimeType },
|
||||
];
|
||||
// Read as image (binary)
|
||||
const buffer = await readFile(absolutePath);
|
||||
const base64 = buffer.toString("base64");
|
||||
|
||||
if (autoResizeImages) {
|
||||
// Resize image if needed
|
||||
const resized = await resizeImage({ type: "image", data: base64, mimeType });
|
||||
const dimensionNote = formatDimensionNote(resized);
|
||||
|
||||
let textNote = `Read image file [${resized.mimeType}]`;
|
||||
if (dimensionNote) {
|
||||
textNote += `\n${dimensionNote}`;
|
||||
}
|
||||
|
||||
content = [
|
||||
{ type: "text", text: textNote },
|
||||
{ type: "image", data: resized.data, mimeType: resized.mimeType },
|
||||
];
|
||||
} else {
|
||||
content = [
|
||||
{ type: "text", text: `Read image file [${mimeType}]` },
|
||||
{ type: "image", data: base64, mimeType },
|
||||
];
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Read as text
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ export interface SettingsConfig {
|
|||
autoCompact: boolean;
|
||||
showImages: boolean;
|
||||
autoResizeImages: boolean;
|
||||
blockImages: boolean;
|
||||
steeringMode: "all" | "one-at-a-time";
|
||||
followUpMode: "all" | "one-at-a-time";
|
||||
thinkingLevel: ThinkingLevel;
|
||||
|
|
@ -40,6 +41,7 @@ export interface SettingsCallbacks {
|
|||
onAutoCompactChange: (enabled: boolean) => void;
|
||||
onShowImagesChange: (enabled: boolean) => void;
|
||||
onAutoResizeImagesChange: (enabled: boolean) => void;
|
||||
onBlockImagesChange: (blocked: boolean) => void;
|
||||
onSteeringModeChange: (mode: "all" | "one-at-a-time") => void;
|
||||
onFollowUpModeChange: (mode: "all" | "one-at-a-time") => void;
|
||||
onThinkingLevelChange: (level: ThinkingLevel) => void;
|
||||
|
|
@ -243,6 +245,16 @@ export class SettingsSelectorComponent extends Container {
|
|||
values: ["true", "false"],
|
||||
});
|
||||
|
||||
// Block images toggle (always available, insert after auto-resize-images)
|
||||
const autoResizeIndex = items.findIndex((item) => item.id === "auto-resize-images");
|
||||
items.splice(autoResizeIndex + 1, 0, {
|
||||
id: "block-images",
|
||||
label: "Block images",
|
||||
description: "Prevent images from being sent to LLM providers (restart session for full effect)",
|
||||
currentValue: config.blockImages ? "true" : "false",
|
||||
values: ["true", "false"],
|
||||
});
|
||||
|
||||
// Add borders
|
||||
this.addChild(new DynamicBorder());
|
||||
|
||||
|
|
@ -261,6 +273,9 @@ export class SettingsSelectorComponent extends Container {
|
|||
case "auto-resize-images":
|
||||
callbacks.onAutoResizeImagesChange(newValue === "true");
|
||||
break;
|
||||
case "block-images":
|
||||
callbacks.onBlockImagesChange(newValue === "true");
|
||||
break;
|
||||
case "steering-mode":
|
||||
callbacks.onSteeringModeChange(newValue as "all" | "one-at-a-time");
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -1936,6 +1936,7 @@ export class InteractiveMode {
|
|||
autoCompact: this.session.autoCompactionEnabled,
|
||||
showImages: this.settingsManager.getShowImages(),
|
||||
autoResizeImages: this.settingsManager.getImageAutoResize(),
|
||||
blockImages: this.settingsManager.getBlockImages(),
|
||||
steeringMode: this.session.steeringMode,
|
||||
followUpMode: this.session.followUpMode,
|
||||
thinkingLevel: this.session.thinkingLevel,
|
||||
|
|
@ -1962,6 +1963,9 @@ export class InteractiveMode {
|
|||
onAutoResizeImagesChange: (enabled) => {
|
||||
this.settingsManager.setImageAutoResize(enabled);
|
||||
},
|
||||
onBlockImagesChange: (blocked) => {
|
||||
this.settingsManager.setBlockImages(blocked);
|
||||
},
|
||||
onSteeringModeChange: (mode) => {
|
||||
this.session.setSteeringMode(mode);
|
||||
},
|
||||
|
|
|
|||
147
packages/coding-agent/test/block-images.test.ts
Normal file
147
packages/coding-agent/test/block-images.test.ts
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
import { mkdirSync, rmSync, writeFileSync } from "fs";
|
||||
import { tmpdir } from "os";
|
||||
import { join } from "path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { processFileArguments } from "../src/cli/file-processor.js";
|
||||
import { SettingsManager } from "../src/core/settings-manager.js";
|
||||
import { createReadTool } from "../src/core/tools/read.js";
|
||||
|
||||
// 1x1 red PNG image as base64 (smallest valid PNG)
|
||||
const TINY_PNG_BASE64 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==";
|
||||
|
||||
describe("blockImages setting", () => {
|
||||
describe("SettingsManager", () => {
|
||||
it("should default blockImages to false", () => {
|
||||
const manager = SettingsManager.inMemory({});
|
||||
expect(manager.getBlockImages()).toBe(false);
|
||||
});
|
||||
|
||||
it("should return true when blockImages is set to true", () => {
|
||||
const manager = SettingsManager.inMemory({ images: { blockImages: true } });
|
||||
expect(manager.getBlockImages()).toBe(true);
|
||||
});
|
||||
|
||||
it("should persist blockImages setting via setBlockImages", () => {
|
||||
const manager = SettingsManager.inMemory({});
|
||||
expect(manager.getBlockImages()).toBe(false);
|
||||
|
||||
manager.setBlockImages(true);
|
||||
expect(manager.getBlockImages()).toBe(true);
|
||||
|
||||
manager.setBlockImages(false);
|
||||
expect(manager.getBlockImages()).toBe(false);
|
||||
});
|
||||
|
||||
it("should handle blockImages alongside autoResize", () => {
|
||||
const manager = SettingsManager.inMemory({
|
||||
images: { autoResize: true, blockImages: true },
|
||||
});
|
||||
expect(manager.getImageAutoResize()).toBe(true);
|
||||
expect(manager.getBlockImages()).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Read tool", () => {
|
||||
let testDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
testDir = join(tmpdir(), `block-images-test-${Date.now()}`);
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("should return text message when blockImages is true", async () => {
|
||||
// Create test image
|
||||
const imagePath = join(testDir, "test.png");
|
||||
writeFileSync(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
|
||||
const tool = createReadTool(testDir, { blockImages: true });
|
||||
const result = await tool.execute("test-1", { path: imagePath });
|
||||
|
||||
expect(result.content).toHaveLength(1);
|
||||
expect(result.content[0].type).toBe("text");
|
||||
const textContent = result.content[0] as { type: "text"; text: string };
|
||||
expect(textContent.text).toContain("Image reading is disabled");
|
||||
expect(textContent.text).toContain("blockImages");
|
||||
});
|
||||
|
||||
it("should return image content when blockImages is false", async () => {
|
||||
// Create test image
|
||||
const imagePath = join(testDir, "test.png");
|
||||
writeFileSync(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
|
||||
const tool = createReadTool(testDir, { blockImages: false });
|
||||
const result = await tool.execute("test-2", { path: imagePath });
|
||||
|
||||
// Should have text note + image content
|
||||
expect(result.content.length).toBeGreaterThanOrEqual(1);
|
||||
const hasImage = result.content.some((c) => c.type === "image");
|
||||
expect(hasImage).toBe(true);
|
||||
});
|
||||
|
||||
it("should read text files normally even when blockImages is true", async () => {
|
||||
// Create test text file
|
||||
const textPath = join(testDir, "test.txt");
|
||||
writeFileSync(textPath, "Hello, world!");
|
||||
|
||||
const tool = createReadTool(testDir, { blockImages: true });
|
||||
const result = await tool.execute("test-3", { path: textPath });
|
||||
|
||||
expect(result.content).toHaveLength(1);
|
||||
expect(result.content[0].type).toBe("text");
|
||||
const textContent = result.content[0] as { type: "text"; text: string };
|
||||
expect(textContent.text).toContain("Hello, world!");
|
||||
});
|
||||
});
|
||||
|
||||
describe("processFileArguments", () => {
|
||||
let testDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
testDir = join(tmpdir(), `block-images-process-test-${Date.now()}`);
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("should skip image files when blockImages is true", async () => {
|
||||
// Create test image
|
||||
const imagePath = join(testDir, "test.png");
|
||||
writeFileSync(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
|
||||
const result = await processFileArguments([imagePath], { blockImages: true });
|
||||
|
||||
expect(result.images).toHaveLength(0);
|
||||
// Text should be empty since image was skipped
|
||||
expect(result.text).toBe("");
|
||||
});
|
||||
|
||||
it("should include image files when blockImages is false", async () => {
|
||||
// Create test image
|
||||
const imagePath = join(testDir, "test.png");
|
||||
writeFileSync(imagePath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
|
||||
const result = await processFileArguments([imagePath], { blockImages: false });
|
||||
|
||||
expect(result.images).toHaveLength(1);
|
||||
expect(result.images[0].type).toBe("image");
|
||||
});
|
||||
|
||||
it("should process text files normally when blockImages is true", async () => {
|
||||
// Create test text file
|
||||
const textPath = join(testDir, "test.txt");
|
||||
writeFileSync(textPath, "Hello, world!");
|
||||
|
||||
const result = await processFileArguments([textPath], { blockImages: true });
|
||||
|
||||
expect(result.images).toHaveLength(0);
|
||||
expect(result.text).toContain("Hello, world!");
|
||||
});
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue