add inline image rendering for terminals with graphics support

packages/coding-agent/src/modes/interactive/components/tool-execution.ts

@@ -1,5 +1,13 @@
 import * as os from "node:os";
-import { Container, Spacer, Text } from "@mariozechner/pi-tui";
+import {
+	Container,
+	getCapabilities,
+	getImageDimensions,
+	Image,
+	imageFallback,
+	Spacer,
+	Text,
+} from "@mariozechner/pi-tui";
 import stripAnsi from "strip-ansi";
 import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize } from "../../../core/tools/truncate.js";
 import { theme } from "../theme/theme.js";
@@ -27,6 +35,7 @@ function replaceTabs(text: string): string {
  */
 export class ToolExecutionComponent extends Container {
 	private contentText: Text;
+	private imageComponents: Image[] = [];
 	private toolName: string;
 	private args: any;
 	private expanded = false;
@@ -41,7 +50,6 @@ export class ToolExecutionComponent extends Container {
 		this.toolName = toolName;
 		this.args = args;
 		this.addChild(new Spacer(1));
-		// Content with colored background and padding
 		this.contentText = new Text("", 1, 1, (text: string) => theme.bg("toolPendingBg", text));
 		this.addChild(this.contentText);
 		this.updateDisplay();
@@ -75,32 +83,54 @@ export class ToolExecutionComponent extends Container {
 
 		this.contentText.setCustomBgFn(bgFn);
 		this.contentText.setText(this.formatToolExecution());
+
+		for (const img of this.imageComponents) {
+			this.removeChild(img);
+		}
+		this.imageComponents = [];
+
+		if (this.result) {
+			const imageBlocks = this.result.content?.filter((c: any) => c.type === "image") || [];
+			const caps = getCapabilities();
+
+			for (const img of imageBlocks) {
+				if (caps.images && img.data && img.mimeType) {
+					const imageComponent = new Image(
+						img.data,
+						img.mimeType,
+						{ fallbackColor: (s: string) => theme.fg("toolOutput", s) },
+						{ maxWidthCells: 60 },
+					);
+					this.imageComponents.push(imageComponent);
+					this.addChild(imageComponent);
+				}
+			}
+		}
 	}
 
 	private getTextOutput(): string {
 		if (!this.result) return "";
 
-		// Extract text from content blocks
 		const textBlocks = this.result.content?.filter((c: any) => c.type === "text") || [];
 		const imageBlocks = this.result.content?.filter((c: any) => c.type === "image") || [];
 
-		// Strip ANSI codes and carriage returns from raw output
-		// (bash may emit colors/formatting, and Windows may include \r)
 		let output = textBlocks
 			.map((c: any) => {
 				let text = stripAnsi(c.text || "").replace(/\r/g, "");
-				// stripAnsi misses some escape sequences like standalone ESC \ (String Terminator)
-				// and leaves orphaned fragments from malformed sequences (e.g. TUI output captured to file)
-				// Clean up: remove ESC + any following char, and control chars except newline/tab
 				text = text.replace(/\x1b./g, "");
 				text = text.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]/g, "");
 				return text;
 			})
 			.join("\n");
 
-		// Add indicator for images
-		if (imageBlocks.length > 0) {
-			const imageIndicators = imageBlocks.map((img: any) => `[Image: ${img.mimeType}]`).join("\n");
+		const caps = getCapabilities();
+		if (imageBlocks.length > 0 && !caps.images) {
+			const imageIndicators = imageBlocks
+				.map((img: any) => {
+					const dims = img.data ? (getImageDimensions(img.data, img.mimeType) ?? undefined) : undefined;
+					return imageFallback(img.mimeType, dims);
+				})
+				.join("\n");
 			output = output ? `${output}\n${imageIndicators}` : imageIndicators;
 		}
 

packages/tui/src/components/image.ts

@@ -0,0 +1,78 @@
+import {
+	getCapabilities,
+	getImageDimensions,
+	type ImageDimensions,
+	imageFallback,
+	renderImage,
+} from "../terminal-image.js";
+import type { Component } from "../tui.js";
+
+export interface ImageTheme {
+	fallbackColor: (str: string) => string;
+}
+
+export interface ImageOptions {
+	maxWidthCells?: number;
+	maxHeightCells?: number;
+	filename?: string;
+}
+
+export class Image implements Component {
+	private base64Data: string;
+	private mimeType: string;
+	private dimensions: ImageDimensions;
+	private theme: ImageTheme;
+	private options: ImageOptions;
+
+	private cachedLines?: string[];
+	private cachedWidth?: number;
+
+	constructor(
+		base64Data: string,
+		mimeType: string,
+		theme: ImageTheme,
+		options: ImageOptions = {},
+		dimensions?: ImageDimensions,
+	) {
+		this.base64Data = base64Data;
+		this.mimeType = mimeType;
+		this.theme = theme;
+		this.options = options;
+		this.dimensions = dimensions || getImageDimensions(base64Data, mimeType) || { widthPx: 800, heightPx: 600 };
+	}
+
+	invalidate(): void {
+		this.cachedLines = undefined;
+		this.cachedWidth = undefined;
+	}
+
+	render(width: number): string[] {
+		if (this.cachedLines && this.cachedWidth === width) {
+			return this.cachedLines;
+		}
+
+		const maxWidth = Math.min(width - 2, this.options.maxWidthCells ?? 60);
+
+		const caps = getCapabilities();
+		let lines: string[];
+
+		if (caps.images) {
+			const result = renderImage(this.base64Data, this.dimensions, { maxWidthCells: maxWidth });
+
+			if (result) {
+				lines = [result.sequence];
+			} else {
+				const fallback = imageFallback(this.mimeType, this.dimensions, this.options.filename);
+				lines = [this.theme.fallbackColor(fallback)];
+			}
+		} else {
+			const fallback = imageFallback(this.mimeType, this.dimensions, this.options.filename);
+			lines = [this.theme.fallbackColor(fallback)];
+		}
+
+		this.cachedLines = lines;
+		this.cachedWidth = width;
+
+		return lines;
+	}
+}

packages/tui/src/index.ts

@@ -9,6 +9,7 @@ export {
 } from "./autocomplete.js";
 // Components
 export { Editor, type EditorTheme } from "./components/editor.js";
+export { Image, type ImageOptions, type ImageTheme } from "./components/image.js";
 export { Input } from "./components/input.js";
 export { Loader } from "./components/loader.js";
 export { type DefaultTextStyle, Markdown, type MarkdownTheme } from "./components/markdown.js";
@@ -18,6 +19,27 @@ export { Text } from "./components/text.js";
 export { TruncatedText } from "./components/truncated-text.js";
 // Terminal interface and implementations
 export { ProcessTerminal, type Terminal } from "./terminal.js";
+// Terminal image support
+export {
+	type CellDimensions,
+	calculateImageRows,
+	detectCapabilities,
+	encodeITerm2,
+	encodeKitty,
+	getCapabilities,
+	getGifDimensions,
+	getImageDimensions,
+	getJpegDimensions,
+	getPngDimensions,
+	getWebpDimensions,
+	type ImageDimensions,
+	type ImageProtocol,
+	type ImageRenderOptions,
+	imageFallback,
+	renderImage,
+	resetCapabilitiesCache,
+	type TerminalCapabilities,
+} from "./terminal-image.js";
 export { type Component, Container, TUI } from "./tui.js";
 // Utilities
 export { truncateToWidth, visibleWidth } from "./utils.js";

packages/tui/src/terminal-image.ts

@@ -0,0 +1,330 @@
+export type ImageProtocol = "kitty" | "iterm2" | null;
+
+export interface TerminalCapabilities {
+	images: ImageProtocol;
+	trueColor: boolean;
+	hyperlinks: boolean;
+}
+
+export interface CellDimensions {
+	widthPx: number;
+	heightPx: number;
+}
+
+export interface ImageDimensions {
+	widthPx: number;
+	heightPx: number;
+}
+
+export interface ImageRenderOptions {
+	maxWidthCells?: number;
+	maxHeightCells?: number;
+	preserveAspectRatio?: boolean;
+}
+
+let cachedCapabilities: TerminalCapabilities | null = null;
+
+export function detectCapabilities(): TerminalCapabilities {
+	const termProgram = process.env.TERM_PROGRAM?.toLowerCase() || "";
+	const term = process.env.TERM?.toLowerCase() || "";
+	const colorTerm = process.env.COLORTERM?.toLowerCase() || "";
+
+	if (process.env.KITTY_WINDOW_ID || termProgram === "kitty") {
+		return { images: "kitty", trueColor: true, hyperlinks: true };
+	}
+
+	if (termProgram === "ghostty" || term.includes("ghostty")) {
+		return { images: "kitty", trueColor: true, hyperlinks: true };
+	}
+
+	if (process.env.WEZTERM_PANE || termProgram === "wezterm") {
+		return { images: "kitty", trueColor: true, hyperlinks: true };
+	}
+
+	if (process.env.ITERM_SESSION_ID || termProgram === "iterm.app") {
+		return { images: "iterm2", trueColor: true, hyperlinks: true };
+	}
+
+	if (termProgram === "vscode") {
+		return { images: null, trueColor: true, hyperlinks: true };
+	}
+
+	if (termProgram === "alacritty") {
+		return { images: null, trueColor: true, hyperlinks: true };
+	}
+
+	const trueColor = colorTerm === "truecolor" || colorTerm === "24bit";
+	return { images: null, trueColor, hyperlinks: true };
+}
+
+export function getCapabilities(): TerminalCapabilities {
+	if (!cachedCapabilities) {
+		cachedCapabilities = detectCapabilities();
+	}
+	return cachedCapabilities;
+}
+
+export function resetCapabilitiesCache(): void {
+	cachedCapabilities = null;
+}
+
+export function encodeKitty(
+	base64Data: string,
+	options: {
+		columns?: number;
+		rows?: number;
+		imageId?: number;
+	} = {},
+): string {
+	const CHUNK_SIZE = 4096;
+
+	const params: string[] = ["a=T", "f=100", "q=2"];
+
+	if (options.columns) params.push(`c=${options.columns}`);
+	if (options.rows) params.push(`r=${options.rows}`);
+	if (options.imageId) params.push(`i=${options.imageId}`);
+
+	if (base64Data.length <= CHUNK_SIZE) {
+		return `\x1b_G${params.join(",")};${base64Data}\x1b\\`;
+	}
+
+	const chunks: string[] = [];
+	let offset = 0;
+	let isFirst = true;
+
+	while (offset < base64Data.length) {
+		const chunk = base64Data.slice(offset, offset + CHUNK_SIZE);
+		const isLast = offset + CHUNK_SIZE >= base64Data.length;
+
+		if (isFirst) {
+			chunks.push(`\x1b_G${params.join(",")},m=1;${chunk}\x1b\\`);
+			isFirst = false;
+		} else if (isLast) {
+			chunks.push(`\x1b_Gm=0;${chunk}\x1b\\`);
+		} else {
+			chunks.push(`\x1b_Gm=1;${chunk}\x1b\\`);
+		}
+
+		offset += CHUNK_SIZE;
+	}
+
+	return chunks.join("");
+}
+
+export function encodeITerm2(
+	base64Data: string,
+	options: {
+		width?: number | string;
+		height?: number | string;
+		name?: string;
+		preserveAspectRatio?: boolean;
+		inline?: boolean;
+	} = {},
+): string {
+	const params: string[] = [`inline=${options.inline !== false ? 1 : 0}`];
+
+	if (options.width !== undefined) params.push(`width=${options.width}`);
+	if (options.height !== undefined) params.push(`height=${options.height}`);
+	if (options.name) {
+		const nameBase64 = Buffer.from(options.name).toString("base64");
+		params.push(`name=${nameBase64}`);
+	}
+	if (options.preserveAspectRatio === false) {
+		params.push("preserveAspectRatio=0");
+	}
+
+	return `\x1b]1337;File=${params.join(";")}:${base64Data}\x07`;
+}
+
+export function calculateImageRows(
+	imageDimensions: ImageDimensions,
+	targetWidthCells: number,
+	cellDimensions: CellDimensions = { widthPx: 9, heightPx: 18 },
+): number {
+	const targetWidthPx = targetWidthCells * cellDimensions.widthPx;
+	const scale = targetWidthPx / imageDimensions.widthPx;
+	const scaledHeightPx = imageDimensions.heightPx * scale;
+	const rows = Math.ceil(scaledHeightPx / cellDimensions.heightPx);
+	return Math.max(1, rows);
+}
+
+export function getPngDimensions(base64Data: string): ImageDimensions | null {
+	try {
+		const buffer = Buffer.from(base64Data, "base64");
+
+		if (buffer.length < 24) {
+			return null;
+		}
+
+		if (buffer[0] !== 0x89 || buffer[1] !== 0x50 || buffer[2] !== 0x4e || buffer[3] !== 0x47) {
+			return null;
+		}
+
+		const width = buffer.readUInt32BE(16);
+		const height = buffer.readUInt32BE(20);
+
+		return { widthPx: width, heightPx: height };
+	} catch {
+		return null;
+	}
+}
+
+export function getJpegDimensions(base64Data: string): ImageDimensions | null {
+	try {
+		const buffer = Buffer.from(base64Data, "base64");
+
+		if (buffer.length < 2) {
+			return null;
+		}
+
+		if (buffer[0] !== 0xff || buffer[1] !== 0xd8) {
+			return null;
+		}
+
+		let offset = 2;
+		while (offset < buffer.length - 9) {
+			if (buffer[offset] !== 0xff) {
+				offset++;
+				continue;
+			}
+
+			const marker = buffer[offset + 1];
+
+			if (marker >= 0xc0 && marker <= 0xc2) {
+				const height = buffer.readUInt16BE(offset + 5);
+				const width = buffer.readUInt16BE(offset + 7);
+				return { widthPx: width, heightPx: height };
+			}
+
+			if (offset + 3 >= buffer.length) {
+				return null;
+			}
+			const length = buffer.readUInt16BE(offset + 2);
+			if (length < 2) {
+				return null;
+			}
+			offset += 2 + length;
+		}
+
+		return null;
+	} catch {
+		return null;
+	}
+}
+
+export function getGifDimensions(base64Data: string): ImageDimensions | null {
+	try {
+		const buffer = Buffer.from(base64Data, "base64");
+
+		if (buffer.length < 10) {
+			return null;
+		}
+
+		const sig = buffer.slice(0, 6).toString("ascii");
+		if (sig !== "GIF87a" && sig !== "GIF89a") {
+			return null;
+		}
+
+		const width = buffer.readUInt16LE(6);
+		const height = buffer.readUInt16LE(8);
+
+		return { widthPx: width, heightPx: height };
+	} catch {
+		return null;
+	}
+}
+
+export function getWebpDimensions(base64Data: string): ImageDimensions | null {
+	try {
+		const buffer = Buffer.from(base64Data, "base64");
+
+		if (buffer.length < 30) {
+			return null;
+		}
+
+		const riff = buffer.slice(0, 4).toString("ascii");
+		const webp = buffer.slice(8, 12).toString("ascii");
+		if (riff !== "RIFF" || webp !== "WEBP") {
+			return null;
+		}
+
+		const chunk = buffer.slice(12, 16).toString("ascii");
+		if (chunk === "VP8 ") {
+			if (buffer.length < 30) return null;
+			const width = buffer.readUInt16LE(26) & 0x3fff;
+			const height = buffer.readUInt16LE(28) & 0x3fff;
+			return { widthPx: width, heightPx: height };
+		} else if (chunk === "VP8L") {
+			if (buffer.length < 25) return null;
+			const bits = buffer.readUInt32LE(21);
+			const width = (bits & 0x3fff) + 1;
+			const height = ((bits >> 14) & 0x3fff) + 1;
+			return { widthPx: width, heightPx: height };
+		} else if (chunk === "VP8X") {
+			if (buffer.length < 30) return null;
+			const width = (buffer[24] | (buffer[25] << 8) | (buffer[26] << 16)) + 1;
+			const height = (buffer[27] | (buffer[28] << 8) | (buffer[29] << 16)) + 1;
+			return { widthPx: width, heightPx: height };
+		}
+
+		return null;
+	} catch {
+		return null;
+	}
+}
+
+export function getImageDimensions(base64Data: string, mimeType: string): ImageDimensions | null {
+	if (mimeType === "image/png") {
+		return getPngDimensions(base64Data);
+	}
+	if (mimeType === "image/jpeg") {
+		return getJpegDimensions(base64Data);
+	}
+	if (mimeType === "image/gif") {
+		return getGifDimensions(base64Data);
+	}
+	if (mimeType === "image/webp") {
+		return getWebpDimensions(base64Data);
+	}
+	return null;
+}
+
+export function renderImage(
+	base64Data: string,
+	imageDimensions: ImageDimensions,
+	options: ImageRenderOptions = {},
+): { sequence: string; rows: number } | null {
+	const caps = getCapabilities();
+
+	if (!caps.images) {
+		return null;
+	}
+
+	const maxWidth = options.maxWidthCells ?? 80;
+	const cellDims: CellDimensions = { widthPx: 9, heightPx: 18 };
+	const rows = calculateImageRows(imageDimensions, maxWidth, cellDims);
+
+	if (caps.images === "kitty") {
+		const sequence = encodeKitty(base64Data, { columns: maxWidth });
+		return { sequence, rows };
+	}
+
+	if (caps.images === "iterm2") {
+		const sequence = encodeITerm2(base64Data, {
+			width: maxWidth,
+			height: "auto",
+			preserveAspectRatio: options.preserveAspectRatio ?? true,
+		});
+		return { sequence, rows };
+	}
+
+	return null;
+}
+
+export function imageFallback(mimeType: string, dimensions?: ImageDimensions, filename?: string): string {
+	const parts: string[] = [];
+	if (filename) parts.push(filename);
+	parts.push(`[${mimeType}]`);
+	if (dimensions) parts.push(`${dimensions.widthPx}x${dimensions.heightPx}`);
+	return `[Image: ${parts.join(" ")}]`;
+}

packages/tui/src/tui.ts

@@ -6,6 +6,7 @@ import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
 import type { Terminal } from "./terminal.js";
+import { getCapabilities } from "./terminal-image.js";
 import { visibleWidth } from "./utils.js";
 
 /**
@@ -121,6 +122,10 @@ export class TUI extends Container {
 		}
 	}
 
+	private containsImage(line: string): boolean {
+		return line.includes("\x1b_G") || line.includes("\x1b]1337;File=");
+	}
+
 	private doRender(): void {
 		const width = this.terminal.columns;
 		const height = this.terminal.rows;
@@ -182,6 +187,30 @@ export class TUI extends Container {
 			return;
 		}
 
+		// Check if we have images - they require special handling to avoid duplication
+		const hasImagesInPrevious = this.previousLines.some((line) => this.containsImage(line));
+		const hasImagesInNew = newLines.some((line) => this.containsImage(line));
+
+		// If images are present and content changed, force full re-render
+		if (hasImagesInPrevious || hasImagesInNew) {
+			let buffer = "\x1b[?2026h"; // Begin synchronized output
+			// For Kitty protocol, delete all images before re-render
+			if (getCapabilities().images === "kitty") {
+				buffer += "\x1b_Ga=d\x1b\\";
+			}
+			buffer += "\x1b[3J\x1b[2J\x1b[H"; // Clear scrollback, screen, and home
+			for (let i = 0; i < newLines.length; i++) {
+				if (i > 0) buffer += "\r\n";
+				buffer += newLines[i];
+			}
+			buffer += "\x1b[?2026l"; // End synchronized output
+			this.terminal.write(buffer);
+			this.cursorRow = newLines.length - 1;
+			this.previousLines = newLines;
+			this.previousWidth = width;
+			return;
+		}
+
 		// Check if firstChanged is outside the viewport
 		// cursorRow is the line where cursor is (0-indexed)
 		// Viewport shows lines from (cursorRow - height + 1) to cursorRow
@@ -222,23 +251,25 @@ export class TUI extends Container {
 		for (let i = firstChanged; i < newLines.length; i++) {
 			if (i > firstChanged) buffer += "\r\n";
 			buffer += "\x1b[2K"; // Clear current line
-			if (visibleWidth(newLines[i]) > width) {
+			const line = newLines[i];
+			const isImageLine = this.containsImage(line);
+			if (!isImageLine && visibleWidth(line) > width) {
 				// Log all lines to crash file for debugging
 				const crashLogPath = path.join(os.homedir(), ".pi", "agent", "pi-crash.log");
 				const crashData = [
 					`Crash at ${new Date().toISOString()}`,
 					`Terminal width: ${width}`,
-					`Line ${i} visible width: ${visibleWidth(newLines[i])}`,
+					`Line ${i} visible width: ${visibleWidth(line)}`,
 					"",
 					"=== All rendered lines ===",
-					...newLines.map((line, idx) => `[${idx}] (w=${visibleWidth(line)}) ${line}`),
+					...newLines.map((l, idx) => `[${idx}] (w=${visibleWidth(l)}) ${l}`),
 					"",
 				].join("\n");
 				fs.mkdirSync(path.dirname(crashLogPath), { recursive: true });
 				fs.writeFileSync(crashLogPath, crashData);
 				throw new Error(`Rendered line ${i} exceeds terminal width. Debug log written to ${crashLogPath}`);
 			}
-			buffer += newLines[i];
+			buffer += line;
 		}
 
 		// If we had more lines before, clear them and move cursor back