diff --git a/packages/coding-agent/src/modes/interactive/components/tool-execution.ts b/packages/coding-agent/src/modes/interactive/components/tool-execution.ts index 305e1927..529eb558 100644 --- a/packages/coding-agent/src/modes/interactive/components/tool-execution.ts +++ b/packages/coding-agent/src/modes/interactive/components/tool-execution.ts @@ -1,5 +1,13 @@ import * as os from "node:os"; -import { Container, Spacer, Text } from "@mariozechner/pi-tui"; +import { + Container, + getCapabilities, + getImageDimensions, + Image, + imageFallback, + Spacer, + Text, +} from "@mariozechner/pi-tui"; import stripAnsi from "strip-ansi"; import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize } from "../../../core/tools/truncate.js"; import { theme } from "../theme/theme.js"; @@ -27,6 +35,7 @@ function replaceTabs(text: string): string { */ export class ToolExecutionComponent extends Container { private contentText: Text; + private imageComponents: Image[] = []; private toolName: string; private args: any; private expanded = false; @@ -41,7 +50,6 @@ export class ToolExecutionComponent extends Container { this.toolName = toolName; this.args = args; this.addChild(new Spacer(1)); - // Content with colored background and padding this.contentText = new Text("", 1, 1, (text: string) => theme.bg("toolPendingBg", text)); this.addChild(this.contentText); this.updateDisplay(); @@ -75,32 +83,54 @@ export class ToolExecutionComponent extends Container { this.contentText.setCustomBgFn(bgFn); this.contentText.setText(this.formatToolExecution()); + + for (const img of this.imageComponents) { + this.removeChild(img); + } + this.imageComponents = []; + + if (this.result) { + const imageBlocks = this.result.content?.filter((c: any) => c.type === "image") || []; + const caps = getCapabilities(); + + for (const img of imageBlocks) { + if (caps.images && img.data && img.mimeType) { + const imageComponent = new Image( + img.data, + img.mimeType, + { fallbackColor: (s: string) => theme.fg("toolOutput", s) }, + { maxWidthCells: 60 }, + ); + this.imageComponents.push(imageComponent); + this.addChild(imageComponent); + } + } + } } private getTextOutput(): string { if (!this.result) return ""; - // Extract text from content blocks const textBlocks = this.result.content?.filter((c: any) => c.type === "text") || []; const imageBlocks = this.result.content?.filter((c: any) => c.type === "image") || []; - // Strip ANSI codes and carriage returns from raw output - // (bash may emit colors/formatting, and Windows may include \r) let output = textBlocks .map((c: any) => { let text = stripAnsi(c.text || "").replace(/\r/g, ""); - // stripAnsi misses some escape sequences like standalone ESC \ (String Terminator) - // and leaves orphaned fragments from malformed sequences (e.g. TUI output captured to file) - // Clean up: remove ESC + any following char, and control chars except newline/tab text = text.replace(/\x1b./g, ""); text = text.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]/g, ""); return text; }) .join("\n"); - // Add indicator for images - if (imageBlocks.length > 0) { - const imageIndicators = imageBlocks.map((img: any) => `[Image: ${img.mimeType}]`).join("\n"); + const caps = getCapabilities(); + if (imageBlocks.length > 0 && !caps.images) { + const imageIndicators = imageBlocks + .map((img: any) => { + const dims = img.data ? (getImageDimensions(img.data, img.mimeType) ?? undefined) : undefined; + return imageFallback(img.mimeType, dims); + }) + .join("\n"); output = output ? `${output}\n${imageIndicators}` : imageIndicators; } diff --git a/packages/tui/src/components/image.ts b/packages/tui/src/components/image.ts new file mode 100644 index 00000000..315038c6 --- /dev/null +++ b/packages/tui/src/components/image.ts @@ -0,0 +1,78 @@ +import { + getCapabilities, + getImageDimensions, + type ImageDimensions, + imageFallback, + renderImage, +} from "../terminal-image.js"; +import type { Component } from "../tui.js"; + +export interface ImageTheme { + fallbackColor: (str: string) => string; +} + +export interface ImageOptions { + maxWidthCells?: number; + maxHeightCells?: number; + filename?: string; +} + +export class Image implements Component { + private base64Data: string; + private mimeType: string; + private dimensions: ImageDimensions; + private theme: ImageTheme; + private options: ImageOptions; + + private cachedLines?: string[]; + private cachedWidth?: number; + + constructor( + base64Data: string, + mimeType: string, + theme: ImageTheme, + options: ImageOptions = {}, + dimensions?: ImageDimensions, + ) { + this.base64Data = base64Data; + this.mimeType = mimeType; + this.theme = theme; + this.options = options; + this.dimensions = dimensions || getImageDimensions(base64Data, mimeType) || { widthPx: 800, heightPx: 600 }; + } + + invalidate(): void { + this.cachedLines = undefined; + this.cachedWidth = undefined; + } + + render(width: number): string[] { + if (this.cachedLines && this.cachedWidth === width) { + return this.cachedLines; + } + + const maxWidth = Math.min(width - 2, this.options.maxWidthCells ?? 60); + + const caps = getCapabilities(); + let lines: string[]; + + if (caps.images) { + const result = renderImage(this.base64Data, this.dimensions, { maxWidthCells: maxWidth }); + + if (result) { + lines = [result.sequence]; + } else { + const fallback = imageFallback(this.mimeType, this.dimensions, this.options.filename); + lines = [this.theme.fallbackColor(fallback)]; + } + } else { + const fallback = imageFallback(this.mimeType, this.dimensions, this.options.filename); + lines = [this.theme.fallbackColor(fallback)]; + } + + this.cachedLines = lines; + this.cachedWidth = width; + + return lines; + } +} diff --git a/packages/tui/src/index.ts b/packages/tui/src/index.ts index f3a41c84..21be0e08 100644 --- a/packages/tui/src/index.ts +++ b/packages/tui/src/index.ts @@ -9,6 +9,7 @@ export { } from "./autocomplete.js"; // Components export { Editor, type EditorTheme } from "./components/editor.js"; +export { Image, type ImageOptions, type ImageTheme } from "./components/image.js"; export { Input } from "./components/input.js"; export { Loader } from "./components/loader.js"; export { type DefaultTextStyle, Markdown, type MarkdownTheme } from "./components/markdown.js"; @@ -18,6 +19,27 @@ export { Text } from "./components/text.js"; export { TruncatedText } from "./components/truncated-text.js"; // Terminal interface and implementations export { ProcessTerminal, type Terminal } from "./terminal.js"; +// Terminal image support +export { + type CellDimensions, + calculateImageRows, + detectCapabilities, + encodeITerm2, + encodeKitty, + getCapabilities, + getGifDimensions, + getImageDimensions, + getJpegDimensions, + getPngDimensions, + getWebpDimensions, + type ImageDimensions, + type ImageProtocol, + type ImageRenderOptions, + imageFallback, + renderImage, + resetCapabilitiesCache, + type TerminalCapabilities, +} from "./terminal-image.js"; export { type Component, Container, TUI } from "./tui.js"; // Utilities export { truncateToWidth, visibleWidth } from "./utils.js"; diff --git a/packages/tui/src/terminal-image.ts b/packages/tui/src/terminal-image.ts new file mode 100644 index 00000000..35daf0d0 --- /dev/null +++ b/packages/tui/src/terminal-image.ts @@ -0,0 +1,330 @@ +export type ImageProtocol = "kitty" | "iterm2" | null; + +export interface TerminalCapabilities { + images: ImageProtocol; + trueColor: boolean; + hyperlinks: boolean; +} + +export interface CellDimensions { + widthPx: number; + heightPx: number; +} + +export interface ImageDimensions { + widthPx: number; + heightPx: number; +} + +export interface ImageRenderOptions { + maxWidthCells?: number; + maxHeightCells?: number; + preserveAspectRatio?: boolean; +} + +let cachedCapabilities: TerminalCapabilities | null = null; + +export function detectCapabilities(): TerminalCapabilities { + const termProgram = process.env.TERM_PROGRAM?.toLowerCase() || ""; + const term = process.env.TERM?.toLowerCase() || ""; + const colorTerm = process.env.COLORTERM?.toLowerCase() || ""; + + if (process.env.KITTY_WINDOW_ID || termProgram === "kitty") { + return { images: "kitty", trueColor: true, hyperlinks: true }; + } + + if (termProgram === "ghostty" || term.includes("ghostty")) { + return { images: "kitty", trueColor: true, hyperlinks: true }; + } + + if (process.env.WEZTERM_PANE || termProgram === "wezterm") { + return { images: "kitty", trueColor: true, hyperlinks: true }; + } + + if (process.env.ITERM_SESSION_ID || termProgram === "iterm.app") { + return { images: "iterm2", trueColor: true, hyperlinks: true }; + } + + if (termProgram === "vscode") { + return { images: null, trueColor: true, hyperlinks: true }; + } + + if (termProgram === "alacritty") { + return { images: null, trueColor: true, hyperlinks: true }; + } + + const trueColor = colorTerm === "truecolor" || colorTerm === "24bit"; + return { images: null, trueColor, hyperlinks: true }; +} + +export function getCapabilities(): TerminalCapabilities { + if (!cachedCapabilities) { + cachedCapabilities = detectCapabilities(); + } + return cachedCapabilities; +} + +export function resetCapabilitiesCache(): void { + cachedCapabilities = null; +} + +export function encodeKitty( + base64Data: string, + options: { + columns?: number; + rows?: number; + imageId?: number; + } = {}, +): string { + const CHUNK_SIZE = 4096; + + const params: string[] = ["a=T", "f=100", "q=2"]; + + if (options.columns) params.push(`c=${options.columns}`); + if (options.rows) params.push(`r=${options.rows}`); + if (options.imageId) params.push(`i=${options.imageId}`); + + if (base64Data.length <= CHUNK_SIZE) { + return `\x1b_G${params.join(",")};${base64Data}\x1b\\`; + } + + const chunks: string[] = []; + let offset = 0; + let isFirst = true; + + while (offset < base64Data.length) { + const chunk = base64Data.slice(offset, offset + CHUNK_SIZE); + const isLast = offset + CHUNK_SIZE >= base64Data.length; + + if (isFirst) { + chunks.push(`\x1b_G${params.join(",")},m=1;${chunk}\x1b\\`); + isFirst = false; + } else if (isLast) { + chunks.push(`\x1b_Gm=0;${chunk}\x1b\\`); + } else { + chunks.push(`\x1b_Gm=1;${chunk}\x1b\\`); + } + + offset += CHUNK_SIZE; + } + + return chunks.join(""); +} + +export function encodeITerm2( + base64Data: string, + options: { + width?: number | string; + height?: number | string; + name?: string; + preserveAspectRatio?: boolean; + inline?: boolean; + } = {}, +): string { + const params: string[] = [`inline=${options.inline !== false ? 1 : 0}`]; + + if (options.width !== undefined) params.push(`width=${options.width}`); + if (options.height !== undefined) params.push(`height=${options.height}`); + if (options.name) { + const nameBase64 = Buffer.from(options.name).toString("base64"); + params.push(`name=${nameBase64}`); + } + if (options.preserveAspectRatio === false) { + params.push("preserveAspectRatio=0"); + } + + return `\x1b]1337;File=${params.join(";")}:${base64Data}\x07`; +} + +export function calculateImageRows( + imageDimensions: ImageDimensions, + targetWidthCells: number, + cellDimensions: CellDimensions = { widthPx: 9, heightPx: 18 }, +): number { + const targetWidthPx = targetWidthCells * cellDimensions.widthPx; + const scale = targetWidthPx / imageDimensions.widthPx; + const scaledHeightPx = imageDimensions.heightPx * scale; + const rows = Math.ceil(scaledHeightPx / cellDimensions.heightPx); + return Math.max(1, rows); +} + +export function getPngDimensions(base64Data: string): ImageDimensions | null { + try { + const buffer = Buffer.from(base64Data, "base64"); + + if (buffer.length < 24) { + return null; + } + + if (buffer[0] !== 0x89 || buffer[1] !== 0x50 || buffer[2] !== 0x4e || buffer[3] !== 0x47) { + return null; + } + + const width = buffer.readUInt32BE(16); + const height = buffer.readUInt32BE(20); + + return { widthPx: width, heightPx: height }; + } catch { + return null; + } +} + +export function getJpegDimensions(base64Data: string): ImageDimensions | null { + try { + const buffer = Buffer.from(base64Data, "base64"); + + if (buffer.length < 2) { + return null; + } + + if (buffer[0] !== 0xff || buffer[1] !== 0xd8) { + return null; + } + + let offset = 2; + while (offset < buffer.length - 9) { + if (buffer[offset] !== 0xff) { + offset++; + continue; + } + + const marker = buffer[offset + 1]; + + if (marker >= 0xc0 && marker <= 0xc2) { + const height = buffer.readUInt16BE(offset + 5); + const width = buffer.readUInt16BE(offset + 7); + return { widthPx: width, heightPx: height }; + } + + if (offset + 3 >= buffer.length) { + return null; + } + const length = buffer.readUInt16BE(offset + 2); + if (length < 2) { + return null; + } + offset += 2 + length; + } + + return null; + } catch { + return null; + } +} + +export function getGifDimensions(base64Data: string): ImageDimensions | null { + try { + const buffer = Buffer.from(base64Data, "base64"); + + if (buffer.length < 10) { + return null; + } + + const sig = buffer.slice(0, 6).toString("ascii"); + if (sig !== "GIF87a" && sig !== "GIF89a") { + return null; + } + + const width = buffer.readUInt16LE(6); + const height = buffer.readUInt16LE(8); + + return { widthPx: width, heightPx: height }; + } catch { + return null; + } +} + +export function getWebpDimensions(base64Data: string): ImageDimensions | null { + try { + const buffer = Buffer.from(base64Data, "base64"); + + if (buffer.length < 30) { + return null; + } + + const riff = buffer.slice(0, 4).toString("ascii"); + const webp = buffer.slice(8, 12).toString("ascii"); + if (riff !== "RIFF" || webp !== "WEBP") { + return null; + } + + const chunk = buffer.slice(12, 16).toString("ascii"); + if (chunk === "VP8 ") { + if (buffer.length < 30) return null; + const width = buffer.readUInt16LE(26) & 0x3fff; + const height = buffer.readUInt16LE(28) & 0x3fff; + return { widthPx: width, heightPx: height }; + } else if (chunk === "VP8L") { + if (buffer.length < 25) return null; + const bits = buffer.readUInt32LE(21); + const width = (bits & 0x3fff) + 1; + const height = ((bits >> 14) & 0x3fff) + 1; + return { widthPx: width, heightPx: height }; + } else if (chunk === "VP8X") { + if (buffer.length < 30) return null; + const width = (buffer[24] | (buffer[25] << 8) | (buffer[26] << 16)) + 1; + const height = (buffer[27] | (buffer[28] << 8) | (buffer[29] << 16)) + 1; + return { widthPx: width, heightPx: height }; + } + + return null; + } catch { + return null; + } +} + +export function getImageDimensions(base64Data: string, mimeType: string): ImageDimensions | null { + if (mimeType === "image/png") { + return getPngDimensions(base64Data); + } + if (mimeType === "image/jpeg") { + return getJpegDimensions(base64Data); + } + if (mimeType === "image/gif") { + return getGifDimensions(base64Data); + } + if (mimeType === "image/webp") { + return getWebpDimensions(base64Data); + } + return null; +} + +export function renderImage( + base64Data: string, + imageDimensions: ImageDimensions, + options: ImageRenderOptions = {}, +): { sequence: string; rows: number } | null { + const caps = getCapabilities(); + + if (!caps.images) { + return null; + } + + const maxWidth = options.maxWidthCells ?? 80; + const cellDims: CellDimensions = { widthPx: 9, heightPx: 18 }; + const rows = calculateImageRows(imageDimensions, maxWidth, cellDims); + + if (caps.images === "kitty") { + const sequence = encodeKitty(base64Data, { columns: maxWidth }); + return { sequence, rows }; + } + + if (caps.images === "iterm2") { + const sequence = encodeITerm2(base64Data, { + width: maxWidth, + height: "auto", + preserveAspectRatio: options.preserveAspectRatio ?? true, + }); + return { sequence, rows }; + } + + return null; +} + +export function imageFallback(mimeType: string, dimensions?: ImageDimensions, filename?: string): string { + const parts: string[] = []; + if (filename) parts.push(filename); + parts.push(`[${mimeType}]`); + if (dimensions) parts.push(`${dimensions.widthPx}x${dimensions.heightPx}`); + return `[Image: ${parts.join(" ")}]`; +} diff --git a/packages/tui/src/tui.ts b/packages/tui/src/tui.ts index ed89cfaf..c3e0479a 100644 --- a/packages/tui/src/tui.ts +++ b/packages/tui/src/tui.ts @@ -6,6 +6,7 @@ import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import type { Terminal } from "./terminal.js"; +import { getCapabilities } from "./terminal-image.js"; import { visibleWidth } from "./utils.js"; /** @@ -121,6 +122,10 @@ export class TUI extends Container { } } + private containsImage(line: string): boolean { + return line.includes("\x1b_G") || line.includes("\x1b]1337;File="); + } + private doRender(): void { const width = this.terminal.columns; const height = this.terminal.rows; @@ -182,6 +187,30 @@ export class TUI extends Container { return; } + // Check if we have images - they require special handling to avoid duplication + const hasImagesInPrevious = this.previousLines.some((line) => this.containsImage(line)); + const hasImagesInNew = newLines.some((line) => this.containsImage(line)); + + // If images are present and content changed, force full re-render + if (hasImagesInPrevious || hasImagesInNew) { + let buffer = "\x1b[?2026h"; // Begin synchronized output + // For Kitty protocol, delete all images before re-render + if (getCapabilities().images === "kitty") { + buffer += "\x1b_Ga=d\x1b\\"; + } + buffer += "\x1b[3J\x1b[2J\x1b[H"; // Clear scrollback, screen, and home + for (let i = 0; i < newLines.length; i++) { + if (i > 0) buffer += "\r\n"; + buffer += newLines[i]; + } + buffer += "\x1b[?2026l"; // End synchronized output + this.terminal.write(buffer); + this.cursorRow = newLines.length - 1; + this.previousLines = newLines; + this.previousWidth = width; + return; + } + // Check if firstChanged is outside the viewport // cursorRow is the line where cursor is (0-indexed) // Viewport shows lines from (cursorRow - height + 1) to cursorRow @@ -222,23 +251,25 @@ export class TUI extends Container { for (let i = firstChanged; i < newLines.length; i++) { if (i > firstChanged) buffer += "\r\n"; buffer += "\x1b[2K"; // Clear current line - if (visibleWidth(newLines[i]) > width) { + const line = newLines[i]; + const isImageLine = this.containsImage(line); + if (!isImageLine && visibleWidth(line) > width) { // Log all lines to crash file for debugging const crashLogPath = path.join(os.homedir(), ".pi", "agent", "pi-crash.log"); const crashData = [ `Crash at ${new Date().toISOString()}`, `Terminal width: ${width}`, - `Line ${i} visible width: ${visibleWidth(newLines[i])}`, + `Line ${i} visible width: ${visibleWidth(line)}`, "", "=== All rendered lines ===", - ...newLines.map((line, idx) => `[${idx}] (w=${visibleWidth(line)}) ${line}`), + ...newLines.map((l, idx) => `[${idx}] (w=${visibleWidth(l)}) ${l}`), "", ].join("\n"); fs.mkdirSync(path.dirname(crashLogPath), { recursive: true }); fs.writeFileSync(crashLogPath, crashData); throw new Error(`Rendered line ${i} exceeds terminal width. Debug log written to ${crashLogPath}`); } - buffer += newLines[i]; + buffer += line; } // If we had more lines before, clear them and move cursor back